def __init__(self, configuration, zoo_keeper, web_socket_clients,
              time_estimate_cache):
     """
     :type configuration: zoom.config.configuration.Configuration
     :type zoo_keeper: kazoo.client.KazooClient
     :type web_socket_clients: list
     """
     self._cache = ApplicationDependenciesMessage()
     self._configuration = configuration
     self._zoo_keeper = zoo_keeper
     self._web_socket_clients = web_socket_clients
     self._time_estimate_cache = time_estimate_cache
     self._message_throttle = MessageThrottle(configuration,
                                              web_socket_clients)
示例#2
0
 def __init__(self, configuration, web_socket_clients):
     """
     :type configuration: zoom.www.config.configuration.Configuration
     :type web_socket_clients: list
     """
     self.configuration = configuration
     self._web_socket_clients = web_socket_clients
     self._message_throttle = MessageThrottle(configuration,
                                              web_socket_clients)
     self.graphite = GraphiteAvailability(
         configuration.graphite_host,
         recheck=configuration.graphite_recheck)
     self.graphite_cache = {}
     self.dependencies = {}
     self.states = {}
示例#3
0
    def __init__(self, configuration, zoo_keeper, web_socket_clients,
                 time_estimate_cache):
        """
        :type configuration: zoom.config.configuration.Configuration
        :type zoo_keeper: zoom.zoo_keeper.ZooKeeper
        :type web_socket_clients: list
        :type time_estimate_cache: zoom.www.cache.time_estimate_cache.TimeEstimateCache
        """
        self._path_to_host_mapping = dict()
        self._configuration = configuration
        self._cache = ApplicationStatesMessage()
        self._cache.set_environment(self._configuration.environment)
        self._zoo_keeper = zoo_keeper
        self._web_socket_clients = web_socket_clients

        self._time_estimate_cache = time_estimate_cache
        self._message_throttle = MessageThrottle(configuration,
                                                 web_socket_clients)
示例#4
0
 def __init__(self, configuration, web_socket_clients):
     """
     :type configuration: zoom.www.config.configuration.Configuration
     :type web_socket_clients: list
     """
     self.configuration = configuration
     self._web_socket_clients = web_socket_clients
     self._message_throttle = MessageThrottle(configuration,
                                              web_socket_clients)
     self.graphite_cache = {}
     self.dependencies = {}
     self.states = {}
示例#5
0
 def __init__(self, configuration, zoo_keeper, web_socket_clients,
              time_estimate_cache):
     """
     :type configuration: zoom.config.configuration.Configuration
     :type zoo_keeper: zoom.www.zoo_keeper.ZooKeeper
     :type web_socket_clients: list
     """
     self._cache = ApplicationDependenciesMessage()
     self._configuration = configuration
     self._zoo_keeper = zoo_keeper
     self._web_socket_clients = web_socket_clients
     self._time_estimate_cache = time_estimate_cache
     self._message_throttle = MessageThrottle(configuration,
                                              web_socket_clients)
    def __init__(self, configuration, zoo_keeper, web_socket_clients,
                 time_estimate_cache):
        """
        :type configuration: zoom.config.configuration.Configuration
        :type zoo_keeper: zoom.zoo_keeper.ZooKeeper
        :type web_socket_clients: list
        :type time_estimate_cache: zoom.www.cache.time_estimate_cache.TimeEstimateCache
        """
        self._path_to_host_mapping = dict()
        self._configuration = configuration
        self._cache = ApplicationStatesMessage()
        self._cache.set_environment(self._configuration.environment)
        self._zoo_keeper = zoo_keeper
        self._web_socket_clients = web_socket_clients

        self._time_estimate_cache = time_estimate_cache
        self._message_throttle = MessageThrottle(configuration,
                                                 web_socket_clients)
class TimeEstimateCache(object):

    def __init__(self, configuration, web_socket_clients):
        """
        :type configuration: zoom.www.config.configuration.Configuration
        :type web_socket_clients: list
        """
        self.configuration = configuration
        self._web_socket_clients = web_socket_clients
        self._message_throttle = MessageThrottle(configuration,
                                                 web_socket_clients)
        self.graphite = GraphiteAvailability(configuration.graphite_host,
                                             recheck=configuration.graphite_recheck)
        self.graphite_cache = {}
        self.dependencies = {}
        self.states = {}

    def start(self):
        self._message_throttle.start()

    def stop(self):
        self._message_throttle.stop()

    def reload(self):
        self.graphite_cache.clear()
        self.load()

    def update_states(self, states):
        """
        :type states: dict
        """
        self.states.update(states)
        self.load(send=True)

    def update_dependencies(self, deps):
        """
        :type deps: dict
        """
        self.dependencies.update(deps)
        self.load(send=True)

    @TimeThis(__file__)
    def load(self, send=False):
        """
        :type send: bool
            Whether to send messages to clients.
        :rtype: zoom.www.messages.global_mode_message.TimeEstimateMessage
        """
        logging.debug("Recomputing Timing Estimates...")
        # Pre-define path in case we except out before it's declared
        path = None
        try:
            message = TimeEstimateMessage()

            cost = self._get_default_data()
            maxpath = "None"
            searchdata = {}

            if self.states and self.graphite.available:
                for path in self.dependencies.iterkeys():
                    data = self._get_max_cost(path, searchdata)
                    if data['max'] > cost['max']:
                        maxpath = path
                    self._get_greatest_cost(cost, data)

            message.update({
                'maxtime': cost['max'],
                'mintime': cost['min'],
                'avetime': cost['ave'],
                'maxpath': maxpath
            })

            if all((send, self.dependencies, self.states)):
                self._message_throttle.add_message(message)

            return message

        except RuntimeError as e:
            message = TimeEstimateMessage()
            logging.exception(e)
            message.update({'error_msg': 'Likely circular dependency on ' + path})
            if all((send, self.dependencies, self.states)):
                self._message_throttle.add_message(message)
            return message

        except Exception as e:
            logging.exception(e)

    def _get_max_cost(self, path, searchdata):
        """
        :type path: str
        :type searchdata: dict
        :rtype: dict
            Example: {'ave': 0, 'max': 0, 'min': 0}
        """
        # init internal search data
        cached_cost = searchdata.get(path, None)
        if cached_cost is not None:
            return cached_cost

        dep_data = self.dependencies.get(path, None)
        greatest_cost = self._get_default_data()

        # take greatest_cost and record the largest cost from all a path's
        # dependencies.
        if dep_data:
            for i in dep_data['dependencies']:
                # expecting: i = {'path': '/foo/bar', 'type': 'baz'}
                if i.get('type').lower() == PredicateType.ZOOKEEPERHASCHILDREN:
                    cached_cost = self._get_max_cost(i.get("path", None), searchdata)
                    self._get_greatest_cost(greatest_cost, cached_cost)

                elif i.get('type').lower() == PredicateType.ZOOKEEPERHASGRANDCHILDREN:
                    grand_path = i.get("path")
                    for key in self.dependencies.iterkeys():
                        if key.lower().startswith(grand_path) and key.lower() != grand_path:
                            cached_cost = self._get_max_cost(key, searchdata)
                            self._get_greatest_cost(greatest_cost, cached_cost)

        # if application is not running, add its cost (from graphite) to the
        # greatest cost of its dependencies
        if (self.states.get(path, None) is not None and
                self.states[path].get('application_status', None) != "running"):
            graphite_data = self.get_graphite_data(path)
            self._add_data(greatest_cost, graphite_data)

        searchdata.update({path: greatest_cost})
        return greatest_cost

    def _get_default_data(self):
        return {'ave': 0, 'max': 0, 'min': 0}

    def _get_greatest_cost(self, d1, d2):
        """
        Update greatest cost values with the greater between the current value
        and the cached value
        """
        for i in ['ave', 'min', 'max']:
            d1[i] = max(d1[i], d2[i])

    def _add_data(self, d1, d2):
        """
        Add an apps own startup time (from graphite) to greatest cost if it is
        not running.
        """
        for i in ['ave', 'min', 'max']:
            d1[i] += d2[i]

    def get_graphite_data(self, path):
        """
        Grab startup times from graphite for a path.
        :type path: str
        :rtype: dict
            Example: {'min': 0, 'max': 0, 'ave': 0}
        """
        if self.graphite_cache.get(path, None) is not None:
            return self.graphite_cache[path]

        try:
            app_path = path.split('/spot/software/state/')[1]
            app_path = app_path.replace('/', '.')
            url = ("http://{0}/render?format=json&from=-7d"
                   "&target=alias(aggregateLine(Infrastructure.startup.{1}.runtime,'max'),'max')"
                   "&target=alias(aggregateLine(Infrastructure.startup.{1}.runtime,'min'),'min')"
                   "&target=alias(aggregateLine(Infrastructure.startup.{1}.runtime,'avg'),'avg')"
                   .format(self.configuration.graphite_host, app_path))

            response = requests.get(url, timeout=.5)

            self.graphite_cache[path] = self._get_default_data()

            if response.status_code == httplib.OK:
                for data in response.json():
                    if "avg" in data['target']:
                        self.graphite_cache[path]['ave'] = data['datapoints'][0][0]
                    elif "max" in data['target']:
                        self.graphite_cache[path]['max'] = data['datapoints'][-1][0]
                    elif "min" in data['target']:
                        self.graphite_cache[path]['min'] = data['datapoints'][0][0]
                    else:
                        logging.warn("Received graphite data {} with unknown "
                                     "target".format(data))

            return self.graphite_cache[path]

        except Exception:
            logging.exception('Error getting startup data from graphite for '
                              'path: {0}'.format(path))
            return self._get_default_data()
示例#8
0
class ApplicationStateCache(object):
    def __init__(self, configuration, zoo_keeper, web_socket_clients,
                 time_estimate_cache):
        """
        :type configuration: zoom.config.configuration.Configuration
        :type zoo_keeper: zoom.zoo_keeper.ZooKeeper
        :type web_socket_clients: list
        :type time_estimate_cache: zoom.www.cache.time_estimate_cache.TimeEstimateCache
        """
        self._path_to_host_mapping = dict()
        self._configuration = configuration
        self._cache = ApplicationStatesMessage()
        self._cache.set_environment(self._configuration.environment)
        self._zoo_keeper = zoo_keeper
        self._web_socket_clients = web_socket_clients

        self._time_estimate_cache = time_estimate_cache
        self._message_throttle = MessageThrottle(configuration,
                                                 web_socket_clients)

    @property
    def host_mapping(self):
        return self._path_to_host_mapping

    def start(self):
        self._message_throttle.start()

    def stop(self):
        self._message_throttle.stop()

    def load(self):
        """
        :rtype: zoom.messages.application_states.ApplicationStatesMessage
        """
        if not len(self._cache):
            self._load()

        return self._cache

    def reload(self):
        self._cache.clear()
        self._on_update_path(self._configuration.application_state_path)

    @TimeThis(__file__)
    def _load(self):
        self._cache.clear()

        self._walk(self._configuration.application_state_path, self._cache)
        logging.info(
            "Application state cache loaded from ZooKeeper {0}".format(
                self._configuration.application_state_path))

        self._time_estimate_cache.update_states(self._cache.application_states)
        self._cache.remove_deletes()

    def _build_default_override_store(self):
        logging.debug('Override storage node not found, creating')
        _template = {}
        self._zoo_keeper.create(self._configuration.override_node,
                                json.dumps(_template),
                                makepath=True)

    def _update_override_info(self, path, override_key, override_value):
        """
        Update zookeeper with override values passed in
        """
        logging.debug(
            "_update_override_info: path: {0}, override_key: {1} override_value: {2}"
            .format(path, override_key, override_value))
        try:
            override_str, stat = self._zoo_keeper.get(
                self._configuration.override_node)
            override_dict = json.loads(override_str)

            update = {override_key: override_value}
            state = override_dict.get(path, {})
            state.update(update)
            override_dict.update({path: state})

            self._zoo_keeper.set(self._configuration.override_node,
                                 json.dumps(override_dict))
        except NoNodeError as err:
            logging.debug('Unable to find {0}, {1}'.format(
                self._configuration.override_node, err))
            self._build_default_override_store()
            self._update_override_info(path, override_key, override_value)

    def manual_update(self, path, key, value):
        """
        Manual override from client of specific value
        :type path: str
        :type key: str
        :type value: str
        """
        state = self._cache.application_states.get(path, None)
        if state is not None:
            self._update_override_info(path, key, value)
            message = ApplicationStatesMessage()
            state[key] = value
            message.update({path: state})
            self._message_throttle.add_message(message)

    @connected_with_return(None)
    def _walk(self, path, result):
        """
        :type path: str
        :type result: zoom.www.messages.application_states.ApplicationStatesMessage
        """
        try:
            children = self._zoo_keeper.get_children(path,
                                                     watch=self._on_update)

            if children:
                for child in children:
                    self._walk(zk_path_join(path, child), result)
            else:
                app_state = self._get_application_state(path)
                result.update(
                    {app_state.configuration_path: app_state.to_dictionary()})

        except NoNodeError:
            result.update({
                path:
                ApplicationState(configuration_path=path,
                                 delete=True).to_dictionary(),
            })

        except Exception:
            logging.exception('An unhandled Exception has occurred while '
                              'running ApplicationStateCache.walk.')

    def _get_app_details(self, path):
        """
        :type path: str
        :rtype: dict, kazoo.protocol.states.ZnodeStat
        """
        raw_data, stat = self._zoo_keeper.get(path, watch=self._on_update)

        data = {}

        if raw_data:
            try:
                data = json.loads(raw_data)
            except ValueError:
                pass

        return data, stat

    def _get_application_state(self, path):
        """
        :type path: str
        :rtype: zoom.entities.application_state.ApplicationState
        """
        data, stat = self._get_app_details(path)

        # persistent node
        if stat.ephemeralOwner == 0:
            # watch node to see if children are created
            self._zoo_keeper.get_children(path, watch=self._on_update)
            host = data.get('host', 'Unknown')
            name = data.get('name', os.path.basename(path))
            agent_path = zk_path_join(self._configuration.agent_state_path,
                                      host)

            # if the agent is down, update state and mode with unknown
            agent_up = bool(
                self._zoo_keeper.exists(agent_path,
                                        watch=self._on_agent_state_update))

            valid = True
            if host in (None, 'Unknown'):
                data['state'] = 'invalid'
                data['mode'] = 'unknown'
                valid = False
            elif not agent_up:
                data['state'] = 'unknown'
                data['mode'] = 'unknown'
                valid = False
            elif agent_up:
                d, s = self._zoo_keeper.get(agent_path)
                registered_comps = json.loads(d).get('components', [])
                if name not in registered_comps:
                    data['state'] = 'invalid'
                    data['mode'] = 'unknown'
                    valid = False

            self._update_mapping(host, {path: valid})

            application_state = ApplicationState(
                application_name=name,
                configuration_path=path,
                application_status=ApplicationStatus.STOPPED,
                application_host=host,
                last_update=stat.last_modified,
                start_stop_time=data.get('start_stop_time', ''),
                error_state=data.get('state', 'unknown'),
                local_mode=data.get('mode', 'unknown'),
                login_user=data.get('login_user', 'Zoom'),
                read_only=data.get('read_only', False),
                last_command=self._get_last_command(data),
                pd_disabled=self._get_existing_attribute(path, 'pd_disabled'),
                grayed=self._get_existing_attribute(path, 'grayed'),
                platform=data.get('platform', 'unknown'),
                restart_count=data.get('restart_count', 0),
                load_times=self._time_estimate_cache.get_graphite_data(path))

        # ephemeral node
        else:
            # watch node to see if it goes away
            self._zoo_keeper.get_children(os.path.dirname(path),
                                          watch=self._on_update)

            host = os.path.basename(path)
            # if it is running, path = /app/path/HOSTNAME
            # need to convert to /app/path to get the app_details
            config_path = os.path.dirname(path)
            parent_data, parent_stat = self._get_app_details(config_path)

            self._update_mapping(host, {config_path: True})

            application_state = ApplicationState(
                application_name=parent_data.get(
                    'name', os.path.basename(config_path)),
                configuration_path=config_path,
                application_status=ApplicationStatus.RUNNING,
                application_host=host,
                last_update=stat.last_modified,
                start_stop_time=parent_data.get('start_stop_time', ''),
                error_state=parent_data.get('state', 'unknown'),
                local_mode=parent_data.get('mode', 'unknown'),
                login_user=parent_data.get('login_user', 'Zoom'),
                read_only=parent_data.get('read_only', False),
                last_command=self._get_last_command(parent_data),
                pd_disabled=self._get_existing_attribute(
                    config_path, 'pd_disabled'),
                grayed=self._get_existing_attribute(config_path, 'grayed'),
                platform=parent_data.get('platform', 'unknown'),
                restart_count=parent_data.get('restart_count', 0),
                load_times=self._time_estimate_cache.get_graphite_data(
                    config_path))

        return application_state

    def _update_mapping(self, host, data):
        """
        :type host: str
        :type data: dict
            {'/some/path', bool}
        """
        d = self._path_to_host_mapping.get(host, {})
        d.update(data)
        self._path_to_host_mapping[host] = d

    def _on_update(self, event):
        """
        Callback to send updates via websocket on application state changes.
        :type event: kazoo.protocol.states.WatchedEvent
        """
        self._on_update_path(event.path)

    def _on_update_path(self, path):
        try:
            message = ApplicationStatesMessage()

            self._walk(path, message)

            self._cache.update(message.application_states)
            self._cache.remove_deletes()

            self._message_throttle.add_message(message)

            self._time_estimate_cache.update_states(
                self._cache.application_states)

        except Exception:
            logging.exception('An unhandled Exception has occurred')

    def _on_agent_state_update(self, event):
        """
        This is to capture when an agent goes down.
        :type event: kazoo.protocol.states.WatchedEvent
        """
        host = os.path.basename(event.path)
        logging.info(
            'Agent on host {0} has changed up/down state.'.format(host))
        paths = self._path_to_host_mapping.get(host, {})
        for p in paths.keys():
            self._on_update_path(p)

    def _get_last_command(self, data):
        """
        :type data: dict
        :rtype: str
        """
        if data.get('state', 'Unknown') in ['starting', 'started']:
            return "Start"
        elif data.get('state', 'Unknown') in ['stopping', 'stopped']:
            return "Stop"
        else:
            return ''

    def _get_existing_attribute(self, path, attr):
        """
        Look for existing value for some value in the app state cache.
        If there is an override value, use that. Else use the existing state.
        :type path: str
        :type attr: str
        """
        state = self._cache.application_states.get(path, None)
        override = {}

        try:
            data, stat = self._zoo_keeper.get(
                self._configuration.override_node)
            override = json.loads(data)
        except (TypeError, ValueError) as err:
            logging.critical('There was a problem returning values from the '
                             'override cache: {0}'.format(err))

        setting = override.get(path, {}).get(attr, None)

        if setting is not None:
            return setting
        elif state is None:
            return None
        else:
            return state.get(attr)
class ApplicationDependencyCache(object):
    def __init__(self, configuration, zoo_keeper, web_socket_clients,
                 time_estimate_cache):
        """
        :type configuration: zoom.config.configuration.Configuration
        :type zoo_keeper: kazoo.client.KazooClient
        :type web_socket_clients: list
        """
        self._cache = ApplicationDependenciesMessage()
        self._configuration = configuration
        self._zoo_keeper = zoo_keeper
        self._web_socket_clients = web_socket_clients
        self._time_estimate_cache = time_estimate_cache
        self._message_throttle = MessageThrottle(configuration,
                                                 web_socket_clients)

    def start(self):
        self._message_throttle.start()

    def stop(self):
        self._message_throttle.stop()

    def load(self):
        """
        :rtype: ApplicationDependenciesMessage
        """
        try:
            if not self._cache.application_dependencies:
                self._load()

            return self._cache
        except Exception:
            logging.exception('An unhandled Exception has occurred')

    def reload(self):
        """
        Clear cache, and re-walk agent config path.
        """
        self._cache.clear()
        logging.info("Application dependency cache cleared")
        self._on_update_path(self._configuration.agent_configuration_path)

    @TimeThis(__file__)
    def _load(self):
        """
        Walk full agent config path to get data. Load self._cache object
        """
        self._cache.clear()

        self._walk(self._configuration.agent_configuration_path, self._cache)
        logging.info(
            "Application dependency cache loaded from ZooKeeper {0}".format(
                self._configuration.agent_configuration_path))

        self._recalc_downstream_dependencies()

        self._time_estimate_cache.update_dependencies(
            self._cache.application_dependencies)

    @connected_with_return(None)
    def _walk(self, path, result):
        """
        :type path: str
        :type result: ApplicationDependenciesMessage
        """
        try:
            children = self._zoo_keeper.get_children(path,
                                                     watch=self._on_update)

            if children:
                for child in children:
                    self._walk(zk_path_join(path, child), result)
            else:
                self._get_application_dependency(path, result)
        except NoNodeError:
            logging.debug('Node at {0} no longer exists.'.format(path))

    def _get_application_dependency(self, path, result):
        """
        Load result object with application dependencies
        :type path: str
        :type result: ApplicationDependenciesMessage
        """
        if self._zoo_keeper.exists(path):
            data, stat = self._zoo_keeper.get(path, watch=self._on_update)
            if not data:
                return

            try:
                root = ElementTree.fromstring(data)

                for node in root.findall('Automation/Component'):

                    app_id = node.attrib.get('id')
                    registrationpath = node.attrib.get('registrationpath',
                                                       None)

                    if registrationpath is None:
                        registrationpath = zk_path_join(
                            self._configuration.application_state_path, app_id)

                    start_action = node.find('Actions/Action[@id="start"]')

                    if start_action is None:
                        logging.warn("No Start Action Found for {0}".format(
                            registrationpath))
                        dependencies = list()
                    else:
                        dependencies = self._parse_dependencies(start_action)

                    data = {
                        "configuration_path": registrationpath,
                        "dependencies": dependencies,
                        "downstream": list()
                    }

                    result.update({registrationpath: data})

            except Exception:
                logging.exception('An unhandled exception occurred')

        else:
            logging.warn("config path does not exist: {0}".format(path))

    def _parse_dependencies(self, action):
        """
        Parse dependencies out of XML
        :type action: xml.etree.ElementTree.Element
        :rtype: list
        """
        # TODO: rename 'path' when it really isn't a path. this is a hack...
        # prev_was_not keeps track of whether the outer class was 'not'
        dependencies = []
        prev_was_not = False
        for predicate in action.iter('Predicate'):
            pred_type = predicate.get('type').lower()
            pred_path = predicate.get('path', None)
            # pred_oper = predicate.get('operational', False)
            pred_oper = bool(
                verify_attribute(predicate, 'operational', none_allowed=True))
            if pred_type == PredicateType.ZOOKEEPERHASCHILDREN:
                dependencies.append({
                    'type': pred_type,
                    'path': pred_path,
                    'operational': pred_oper
                })
                prev_was_not = False
            elif pred_type == PredicateType.ZOOKEEPERHASGRANDCHILDREN:
                dependencies.append({
                    'type': pred_type,
                    'path': pred_path,
                    'operational': pred_oper
                })
                prev_was_not = False
            elif pred_type == PredicateType.ZOOKEEPERGOODUNTILTIME:
                if len(pred_path.split('gut/')) > 1:
                    dependencies.append({
                        'type':
                        pred_type,
                        'path': ("I should be up between: {0}".format(
                            pred_path.split("gut/")[1])),
                        'operational':
                        pred_oper
                    })
                else:
                    logging.debug('Invalid GUT path: {0}'.format(pred_path))
                prev_was_not = False
            elif pred_type == PredicateType.HOLIDAY:
                dependencies.append({
                    'type':
                    pred_type,
                    'path': ("Does NOT run on holidays"
                             if prev_was_not else "Runs on holidays"),
                    'operational':
                    pred_oper
                })
                prev_was_not = False
            elif pred_type == PredicateType.WEEKEND:
                dependencies.append({
                    'type':
                    pred_type,
                    'path': ("Does NOT run on weekends"
                             if prev_was_not else "Runs on weekends"),
                    'operational':
                    pred_oper
                })
                prev_was_not = False
            elif pred_type == PredicateType.TIMEWINDOW:
                begin = predicate.get('begin', None)
                end = predicate.get('end', None)
                weekdays = predicate.get('weekdays', None)
                msg = 'I should be up '
                if begin is not None:
                    msg += 'after: {0} '.format(begin)
                if end is not None:
                    msg += 'until: {0}'.format(end)
                # only send dependency if there is something to send
                if begin is not None or end is not None:
                    dependencies.append({
                        'type': pred_type,
                        'path': msg,
                        'operational': pred_oper
                    })

                # pretend this is a weekend predicate for convenience
                if weekdays is not None:
                    day_range = TimeWindow.parse_range(weekdays)
                    if Weekdays.SATURDAY in day_range or Weekdays.SUNDAY in day_range:
                        wk_msg = 'Runs on weekends'
                    else:
                        wk_msg = 'Does NOT run on weekends'

                    dependencies.append({
                        'type': PredicateType.WEEKEND,
                        'path': wk_msg,
                        'operational': pred_oper
                    })

            elif pred_type == PredicateType.NOT:
                prev_was_not = True

        return dependencies

    @TimeThis(__file__)
    def _recalc_downstream_dependencies(self, tries=0):
        """
        Loop over existing cache and link upstream with downstream elements
        """
        # clear existing downstream
        try:
            for data in self._cache.application_dependencies.itervalues():
                downstream = data.get('downstream')
                del downstream[:]

            dep_copy = self._cache.application_dependencies.copy()
            for path, data in dep_copy.iteritems():
                for key, value in self._cache.application_dependencies.iteritems(
                ):
                    for dep in data.get('dependencies'):
                        if dep['type'] == PredicateType.ZOOKEEPERHASGRANDCHILDREN:
                            if key.startswith(dep['path']):
                                value.get('downstream').append(path)
                        elif dep['type'] == PredicateType.ZOOKEEPERHASCHILDREN:
                            if dep['path'] == key:
                                value.get('downstream').append(path)
        except RuntimeError:
            time.sleep(1)
            tries += 1
            if tries < 3:
                self._recalc_downstream_dependencies(tries=tries)

    def _on_update(self, event):
        """
        Callback to send updates via websocket on application state changes.
        :type event: kazoo.protocol.states.WatchedEvent
        """
        self._on_update_path(event.path)

    def _on_update_path(self, path):
        try:
            message = ApplicationDependenciesMessage()

            self._walk(path, message)

            self._cache.update(message.application_dependencies)

            self._recalc_downstream_dependencies()

            self._message_throttle.add_message(message)

            self._time_estimate_cache.update_dependencies(
                self._cache.application_dependencies)

        except Exception:
            logging.exception('An unhandled Exception has occurred for path: '
                              '{0}'.format(path))
class ApplicationStateCache(object):
    def __init__(self, configuration, zoo_keeper, web_socket_clients,
                 time_estimate_cache):
        """
        :type configuration: zoom.config.configuration.Configuration
        :type zoo_keeper: zoom.zoo_keeper.ZooKeeper
        :type web_socket_clients: list
        :type time_estimate_cache: zoom.www.cache.time_estimate_cache.TimeEstimateCache
        """
        self._path_to_host_mapping = dict()
        self._configuration = configuration
        self._cache = ApplicationStatesMessage()
        self._cache.set_environment(self._configuration.environment)
        self._zoo_keeper = zoo_keeper
        self._web_socket_clients = web_socket_clients

        self._time_estimate_cache = time_estimate_cache
        self._message_throttle = MessageThrottle(configuration,
                                                 web_socket_clients)

    @property
    def host_mapping(self):
        return self._path_to_host_mapping

    def start(self):
        self._message_throttle.start()

    def stop(self):
        self._message_throttle.stop()

    def load(self):
        """
        :rtype: zoom.messages.application_states.ApplicationStatesMessage
        """
        if not len(self._cache):
            self._load()

        return self._cache

    def reload(self):
        self._cache.clear()
        self._on_update_path(self._configuration.application_state_path)

    @TimeThis(__file__)
    def _load(self):
        self._cache.clear()

        self._walk(self._configuration.application_state_path, self._cache)
        logging.info("Application state cache loaded from ZooKeeper {0}"
                     .format(self._configuration.application_state_path))

        self._time_estimate_cache.update_states(
            self._cache.application_states)
        self._cache.remove_deletes()

    def _build_default_override_store(self):
        logging.debug('Override storage node not found, creating')
        _template = {}
        self._zoo_keeper.create(self._configuration.override_node,
                                json.dumps(_template), makepath=True)

    def _update_override_info(self, path, override_key, override_value):
        """
        Update zookeeper with override values passed in
        """
        logging.debug("_update_override_info: path: {0}, override_key: {1} override_value: {2}"
                      .format(path, override_key, override_value))
        try:
            override_str, stat = self._zoo_keeper.get(self._configuration.override_node)
            override_dict = json.loads(override_str)

            update = {override_key: override_value}
            state = override_dict.get(path, {})
            state.update(update)
            override_dict.update({path: state})

            self._zoo_keeper.set(self._configuration.override_node,
                                 json.dumps(override_dict))
        except NoNodeError as err:
            logging.debug('Unable to find {0}, {1}'
                          .format(self._configuration.override_node, err))
            self._build_default_override_store()
            self._update_override_info(path, override_key, override_value)

    def manual_update(self, path, key, value):
        """
        Manual override from client of specific value
        :type path: str
        :type key: str
        :type value: str
        """
        state = self._cache.application_states.get(path, None)
        if state is not None:
            self._update_override_info(path, key, value)
            message = ApplicationStatesMessage()
            state[key] = value
            message.update({path: state})
            self._message_throttle.add_message(message)

    @connected_with_return(None)
    def _walk(self, path, result):
        """
        :type path: str
        :type result: zoom.www.messages.application_states.ApplicationStatesMessage
        """
        try:
            children = self._zoo_keeper.get_children(path, watch=self._on_update)

            if children:
                for child in children:
                    self._walk(zk_path_join(path, child), result)
            else:
                app_state = self._get_application_state(path)
                result.update(
                    {app_state.configuration_path: app_state.to_dictionary()}
                )

        except NoNodeError:
            result.update({path: ApplicationState(configuration_path=path,
                                                  delete=True).to_dictionary(),
            })

        except Exception:
            logging.exception('An unhandled Exception has occurred while '
                              'running ApplicationStateCache.walk.')

    def _get_app_details(self, path):
        """
        :type path: str
        :rtype: dict, kazoo.protocol.states.ZnodeStat
        """
        raw_data, stat = self._zoo_keeper.get(path, watch=self._on_update)

        data = {}

        if raw_data:
            try:
                data = json.loads(raw_data)
            except ValueError:
                pass

        return data, stat

    def _get_application_state(self, path):
        """
        :type path: str
        :rtype: zoom.entities.application_state.ApplicationState
        """
        data, stat = self._get_app_details(path)

        # persistent node
        if stat.ephemeralOwner == 0:
            # watch node to see if children are created
            self._zoo_keeper.get_children(path, watch=self._on_update)
            host = data.get('host', 'Unknown')
            name = data.get('name', os.path.basename(path))
            agent_path = zk_path_join(self._configuration.agent_state_path,
                                      host)

            # if the agent is down, update state and mode with unknown
            agent_up = bool(self._zoo_keeper.exists(
                agent_path,
                watch=self._on_agent_state_update))

            valid = True
            if host in (None, 'Unknown'):
                data['state'] = 'invalid'
                data['mode'] = 'unknown'
                valid = False
            elif not agent_up:
                data['state'] = 'unknown'
                data['mode'] = 'unknown'
                valid = False
            elif agent_up:
                d, s = self._zoo_keeper.get(agent_path)
                registered_comps = json.loads(d).get('components', [])
                if name not in registered_comps:
                    data['state'] = 'invalid'
                    data['mode'] = 'unknown'
                    valid = False

            self._update_mapping(host, {path: valid})

            application_state = ApplicationState(
                application_name=name,
                configuration_path=path,
                application_status=ApplicationStatus.STOPPED,
                application_host=host,
                last_update=stat.last_modified,
                start_stop_time=data.get('start_stop_time', ''),
                error_state=data.get('state', 'unknown'),
                local_mode=data.get('mode', 'unknown'),
                login_user=data.get('login_user', 'Zoom'),
                read_only=data.get('read_only', False),
                last_command=self._get_last_command(data),
                pd_disabled=self._get_existing_attribute(path, 'pd_disabled'),
                grayed=self._get_existing_attribute(path, 'grayed'),
                platform=data.get('platform', 'unknown'),
                restart_count=data.get('restart_count', 0),
                load_times=self._time_estimate_cache.get_graphite_data(path)
            )

        # ephemeral node
        else:
            # watch node to see if it goes away
            self._zoo_keeper.get_children(os.path.dirname(path),
                                          watch=self._on_update)

            host = os.path.basename(path)
            # if it is running, path = /app/path/HOSTNAME
            # need to convert to /app/path to get the app_details
            config_path = os.path.dirname(path)
            parent_data, parent_stat = self._get_app_details(config_path)

            self._update_mapping(host, {config_path: True})

            application_state = ApplicationState(
                application_name=parent_data.get('name',
                                                 os.path.basename(config_path)),
                configuration_path=config_path,
                application_status=ApplicationStatus.RUNNING,
                application_host=host,
                last_update=stat.last_modified,
                start_stop_time=parent_data.get('start_stop_time', ''),
                error_state=parent_data.get('state', 'unknown'),
                local_mode=parent_data.get('mode', 'unknown'),
                login_user=parent_data.get('login_user', 'Zoom'),
                read_only=parent_data.get('read_only', False),
                last_command=self._get_last_command(parent_data),
                pd_disabled=self._get_existing_attribute(config_path, 'pd_disabled'),
                grayed=self._get_existing_attribute(config_path, 'grayed'),
                platform=parent_data.get('platform', 'unknown'),
                restart_count=parent_data.get('restart_count', 0),
                load_times=self._time_estimate_cache.get_graphite_data(config_path)
            )

        return application_state

    def _update_mapping(self, host, data):
        """
        :type host: str
        :type data: dict
            {'/some/path', bool}
        """
        d = self._path_to_host_mapping.get(host, {})
        d.update(data)
        self._path_to_host_mapping[host] = d

    def _on_update(self, event):
        """
        Callback to send updates via websocket on application state changes.
        :type event: kazoo.protocol.states.WatchedEvent
        """
        self._on_update_path(event.path)

    def _on_update_path(self, path):
        try:
            message = ApplicationStatesMessage()

            self._walk(path, message)

            self._cache.update(message.application_states)
            self._cache.remove_deletes()

            self._message_throttle.add_message(message)

            self._time_estimate_cache.update_states(
                self._cache.application_states)

        except Exception:
            logging.exception('An unhandled Exception has occurred')

    def _on_agent_state_update(self, event):
        """
        This is to capture when an agent goes down.
        :type event: kazoo.protocol.states.WatchedEvent
        """
        host = os.path.basename(event.path)
        logging.info('Agent on host {0} has changed up/down state.'.format(host))
        paths = self._path_to_host_mapping.get(host, {})
        for p in paths.keys():
            self._on_update_path(p)

    def _get_last_command(self, data):
        """
        :type data: dict
        :rtype: str
        """
        if data.get('state', 'Unknown') in ['starting', 'started']:
            return "Start"
        elif data.get('state', 'Unknown') in ['stopping', 'stopped']:
            return "Stop"
        else:
            return ''

    def _get_existing_attribute(self, path, attr):
        """
        Look for existing value for some value in the app state cache.
        If there is an override value, use that. Else use the existing state.
        :type path: str
        :type attr: str
        """
        state = self._cache.application_states.get(path, None)
        override = {}

        try:
            data, stat = self._zoo_keeper.get(self._configuration.override_node)
            override = json.loads(data)
        except (TypeError, ValueError) as err:
            logging.critical('There was a problem returning values from the '
                             'override cache: {0}'.format(err))

        setting = override.get(path, {}).get(attr, None)

        if setting is not None:
            return setting
        elif state is None:
            return None
        else:
            return state.get(attr)
示例#11
0
class TimeEstimateCache(object):
    def __init__(self, configuration, web_socket_clients):
        """
        :type configuration: zoom.www.config.configuration.Configuration
        :type web_socket_clients: list
        """
        self.configuration = configuration
        self._web_socket_clients = web_socket_clients
        self._message_throttle = MessageThrottle(configuration,
                                                 web_socket_clients)
        self.graphite = GraphiteAvailability(
            configuration.graphite_host,
            recheck=configuration.graphite_recheck)
        self.graphite_cache = {}
        self.dependencies = {}
        self.states = {}

    def start(self):
        self._message_throttle.start()

    def stop(self):
        self._message_throttle.stop()

    def reload(self):
        self.graphite_cache.clear()
        self.load()

    def update_states(self, states):
        """
        :type states: dict
        """
        self.states.update(states)
        self.load(send=True)

    def update_dependencies(self, deps):
        """
        :type deps: dict
        """
        self.dependencies.update(deps)
        self.load(send=True)

    @TimeThis(__file__)
    def load(self, send=False):
        """
        :type send: bool
            Whether to send messages to clients.
        :rtype: zoom.www.messages.global_mode_message.TimeEstimateMessage
        """
        logging.debug("Recomputing Timing Estimates...")
        # Pre-define path in case we except out before it's declared
        path = None
        try:
            message = TimeEstimateMessage()

            cost = self._get_default_data()
            maxpath = "None"
            searchdata = {}

            if self.states and self.graphite.available:
                for path in self.dependencies.iterkeys():
                    data = self._get_max_cost(path, searchdata)
                    if data['max'] > cost['max']:
                        maxpath = path
                    self._get_greatest_cost(cost, data)

            message.update({
                'maxtime': cost['max'],
                'mintime': cost['min'],
                'avetime': cost['ave'],
                'maxpath': maxpath
            })

            if all((send, self.dependencies, self.states)):
                self._message_throttle.add_message(message)

            return message

        except RuntimeError as e:
            message = TimeEstimateMessage()
            logging.exception(e)
            message.update(
                {'error_msg': 'Likely circular dependency on ' + path})
            if all((send, self.dependencies, self.states)):
                self._message_throttle.add_message(message)
            return message

        except Exception as e:
            logging.exception(e)

    def _get_max_cost(self, path, searchdata):
        """
        :type path: str
        :type searchdata: dict
        :rtype: dict
            Example: {'ave': 0, 'max': 0, 'min': 0}
        """
        # init internal search data
        cached_cost = searchdata.get(path, None)
        if cached_cost is not None:
            return cached_cost

        dep_data = self.dependencies.get(path, None)
        greatest_cost = self._get_default_data()

        # take greatest_cost and record the largest cost from all a path's
        # dependencies.
        if dep_data:
            for i in dep_data['dependencies']:
                # expecting: i = {'path': '/foo/bar', 'type': 'baz'}
                if i.get('type').lower() == PredicateType.ZOOKEEPERHASCHILDREN:
                    cached_cost = self._get_max_cost(i.get("path", None),
                                                     searchdata)
                    self._get_greatest_cost(greatest_cost, cached_cost)

                elif i.get('type').lower(
                ) == PredicateType.ZOOKEEPERHASGRANDCHILDREN:
                    grand_path = i.get("path")
                    for key in self.dependencies.iterkeys():
                        if key.lower().startswith(
                                grand_path) and key.lower() != grand_path:
                            cached_cost = self._get_max_cost(key, searchdata)
                            self._get_greatest_cost(greatest_cost, cached_cost)

        # if application is not running, add its cost (from graphite) to the
        # greatest cost of its dependencies
        if (self.states.get(path, None) is not None and self.states[path].get(
                'application_status', None) != "running"):
            graphite_data = self.get_graphite_data(path)
            self._add_data(greatest_cost, graphite_data)

        searchdata.update({path: greatest_cost})
        return greatest_cost

    def _get_default_data(self):
        return {'ave': 0, 'max': 0, 'min': 0}

    def _get_greatest_cost(self, d1, d2):
        """
        Update greatest cost values with the greater between the current value
        and the cached value
        """
        for i in ['ave', 'min', 'max']:
            d1[i] = max(d1[i], d2[i])

    def _add_data(self, d1, d2):
        """
        Add an apps own startup time (from graphite) to greatest cost if it is
        not running.
        """
        for i in ['ave', 'min', 'max']:
            d1[i] += d2[i]

    def get_graphite_data(self, path):
        """
        Grab startup times from graphite for a path.
        :type path: str
        :rtype: dict
            Example: {'min': 0, 'max': 0, 'ave': 0}
        """
        if self.graphite_cache.get(path, None) is not None:
            return self.graphite_cache[path]

        try:
            app_path = path.split('/spot/software/state/')[1]
            app_path = app_path.replace('/', '.')
            url = (
                "http://{0}/render?format=json&from=-7d"
                "&target=alias(aggregateLine(Infrastructure.startup.{1}.runtime,'max'),'max')"
                "&target=alias(aggregateLine(Infrastructure.startup.{1}.runtime,'min'),'min')"
                "&target=alias(aggregateLine(Infrastructure.startup.{1}.runtime,'avg'),'avg')"
                .format(self.configuration.graphite_host, app_path))

            response = requests.get(url, timeout=.5)

            self.graphite_cache[path] = self._get_default_data()

            if response.status_code == httplib.OK:
                for data in response.json():
                    if "avg" in data['target']:
                        self.graphite_cache[path]['ave'] = data['datapoints'][
                            0][0]
                    elif "max" in data['target']:
                        self.graphite_cache[path]['max'] = data['datapoints'][
                            -1][0]
                    elif "min" in data['target']:
                        self.graphite_cache[path]['min'] = data['datapoints'][
                            0][0]
                    else:
                        logging.warn("Received graphite data {} with unknown "
                                     "target".format(data))

            return self.graphite_cache[path]

        except Exception:
            logging.exception('Error getting startup data from graphite for '
                              'path: {0}'.format(path))
            return self._get_default_data()
示例#12
0
class ApplicationStateCache(object):
    def __init__(self, configuration, zoo_keeper, web_socket_clients,
                 time_estimate_cache):
        """
        :type configuration: zoom.config.configuration.Configuration
        :type zoo_keeper: zoom.zoo_keeper.ZooKeeper
        :type web_socket_clients: list
        :type time_estimate_cache: zoom.www.cache.time_estimate_cache.TimeEstimateCache
        """
        self._path_to_host_mapping = dict()
        self._configuration = configuration
        self._cache = ApplicationStatesMessage()
        self._cache.set_environment(self._configuration.environment)
        self._zoo_keeper = zoo_keeper
        self._web_socket_clients = web_socket_clients

        self._time_estimate_cache = time_estimate_cache
        self._message_throttle = MessageThrottle(configuration,
                                                 web_socket_clients)
        self._last_command = None

    def start(self):
        self._message_throttle.start()

    def stop(self):
        self._message_throttle.stop()

    def load(self):
        """
        :rtype: zoom.messages.application_states.ApplicationStatesMessage
        """
        if not len(self._cache):
            self._load()

        return self._cache

    def reload(self):
        self._cache.clear()
        self._on_update_path(self._configuration.application_state_path)

    def _load(self):
        self._cache.clear()

        self._walk(self._configuration.application_state_path, self._cache)
        logging.info("Application state cache loaded from ZooKeeper {0}"
                     .format(self._configuration.application_state_path))

        self._time_estimate_cache.update_states(
            self._cache.application_states)
        self._cache.remove_deletes()

    @connected_with_return(None)
    def _walk(self, path, result):
        """
        :type path: str
        :type result: zoom.www.messages.application_states.ApplicationStatesMessage
        """
        try:
            children = self._zoo_keeper.get_children(path, watch=self._on_update)

            if children:
                for child in children:
                    self._walk(os.path.join(path, child), result)
            else:
                app_state = self._get_application_state(path)
                result.update(
                    {app_state.configuration_path: app_state.to_dictionary()}
                )

        except NoNodeError:
            logging.debug('Node at {0} no longer exists.'.format(path))
            result.update({path: ApplicationState(configuration_path=path,
                                                  delete=True).to_dictionary(),
            })

        except Exception:
            logging.exception('An unhandled Exception has occurred while '
                              'running ApplicationStateCache.walk.')

    def _get_app_details(self, path):
        """
        :type path: str
        :rtype: dict, kazoo.protocol.states.ZnodeStat
        """
        rawData, stat = self._zoo_keeper.get(path, watch=self._on_update)

        data = {}

        if rawData != '':
            try:
                data = json.loads(rawData)
            except ValueError:
                pass

        return data, stat

    def _get_application_state(self, path):
        """
        :type path: str
        :rtype: zoom.entities.application_state.ApplicationState
        """
        data, stat = self._get_app_details(path)

        # persistent node
        if stat.ephemeralOwner == 0:
            # watch node to see if children are created
            self._zoo_keeper.get_children(path, watch=self._on_update)
            host = data.get('host', 'Unknown')
            agent_path = os.path.join(self._configuration.agent_state_path,
                                      host)

            # if the agent is down, update state and mode with unknown
            if (host is None or 
                    not self._zoo_keeper.exists(
                            agent_path,
                            watch=self._on_agent_state_update)):
                data['state'] = 'unknown'
                data['mode'] = 'unknown'
            else:
                self._path_to_host_mapping[host] = path

            application_state = ApplicationState(
                application_name=data.get('name', os.path.basename(path)),
                configuration_path=path,
                application_status=ApplicationStatus.STOPPED,
                application_host=host,
                completion_time=stat.last_modified,
                trigger_time=data.get('trigger_time', ''),
                error_state=data.get('state', 'unknown'),
                local_mode=data.get('mode', 'unknown'),
                login_user=data.get('login_user', 'Zoom'),
                fqdn=data.get('fqdn', host),
                last_command=self._get_last_command(data)
            )

        # ephemeral node
        else:
            # watch node to see if it goes away
            self._zoo_keeper.get_children(os.path.dirname(path),
                                          watch=self._on_update)

            host = os.path.basename(path)
            # if it is running, path = /app/path/HOSTNAME
            # need to convert to /app/path to get the app_details
            config_path = os.path.dirname(path)
            parent_data, parent_stat = self._get_app_details(config_path)

            self._path_to_host_mapping[host] = config_path

            application_state = ApplicationState(
                application_name=parent_data.get('name',
                                                 os.path.basename(config_path)),
                configuration_path=config_path,
                application_status=ApplicationStatus.RUNNING,
                application_host=host,
                completion_time=stat.last_modified,
                trigger_time=parent_data.get('trigger_time', ''),
                error_state=parent_data.get('state', 'unknown'),
                local_mode=parent_data.get('mode', 'unknown'),
                login_user=parent_data.get('login_user', 'Zoom'),
                fqdn=parent_data.get('fqdn', host),
                last_command=self._get_last_command(parent_data)
            )
        return application_state

    def _on_update(self, event):
        """
        Callback to send updates via websocket on application state changes.
        :type event: kazoo.protocol.states.WatchedEvent
        """
        self._on_update_path(event.path)

    def _on_update_path(self, path):
        try:
            message = ApplicationStatesMessage()

            self._walk(path, message)

            self._cache.update(message.application_states)
            self._cache.remove_deletes()

            self._message_throttle.add_message(message)

            self._time_estimate_cache.update_states(
                self._cache.application_states)

        except Exception:
            logging.exception('An unhandled Exception has occurred')

    def _on_agent_state_update(self, event):
        """
        This is to capture when an agent goes down.
        :type event: kazoo.protocol.states.WatchedEvent
        """
        host = os.path.basename(event.path)
        path = self._path_to_host_mapping.get(host, None)
        if path is not None:  # if data is in the cache
            self._on_update_path(path)

    def _get_last_command(self, data):
        if data.get('state', 'Unknown') == 'starting':
            self._last_command = "Start"
        elif data.get('state', 'Unknown') == 'stopping':
            self._last_command = "Stop"
        else:
            pass

        return self._last_command
示例#13
0
class ApplicationDependencyCache(object):
    def __init__(self, configuration, zoo_keeper, web_socket_clients,
                 time_estimate_cache):
        """
        :type configuration: zoom.config.configuration.Configuration
        :type zoo_keeper: zoom.www.zoo_keeper.ZooKeeper
        :type web_socket_clients: list
        """
        self._cache = ApplicationDependenciesMessage()
        self._configuration = configuration
        self._zoo_keeper = zoo_keeper
        self._web_socket_clients = web_socket_clients
        self._time_estimate_cache = time_estimate_cache
        self._message_throttle = MessageThrottle(configuration,
                                                 web_socket_clients)

    def start(self):
        self._message_throttle.start()

    def stop(self):
        self._message_throttle.stop()

    def load(self):
        """
        :rtype: ApplicationDependenciesMessage
        """
        try:
            if not self._cache.application_dependencies:
                self._load()

            return self._cache
        except Exception:
            logging.exception('An unhandled Exception has occurred')

    def reload(self):
        """
        Clear cache, and re-walk agent config path.
        """
        self._cache.clear()
        logging.info("Application dependency cache cleared")
        self._on_update_path(self._configuration.agent_configuration_path)

    def _load(self):
        """
        Walk full agent config path to get data. Load self._cache object
        """
        self._cache.clear()

        self._walk(self._configuration.agent_configuration_path, self._cache)
        logging.info("Application dependency cache loaded from ZooKeeper {0}"
                     .format(self._configuration.agent_configuration_path))

        self._recalc_downstream_dependencies()

        self._time_estimate_cache.update_dependencies(
            self._cache.application_dependencies)

    @connected_with_return(None)
    def _walk(self, path, result):
        """
        :type path: str
        :type result: ApplicationDependenciesMessage
        """
        try:
            children = self._zoo_keeper.get_children(path,
                                                     watch=self._on_update)

            if children:
                for child in children:
                    self._walk(os.path.join(path, child), result)
            else:
                self._get_application_dependency(path, result)
        except NoNodeError:
            logging.debug('Node at {0} no longer exists.'.format(path))

    def _get_application_dependency(self, path, result):
        """
        Load result object with application dependencies
        :type path: str
        :type result: ApplicationDependenciesMessage
        """
        if self._zoo_keeper.exists(path):
            data, stat = self._zoo_keeper.get(path, watch=self._on_update)
            if not data:
                return

            try:
                root = ElementTree.fromstring(data)

                for node in root.findall('Automation/Component'):

                    app_id = node.attrib.get('id')
                    registrationpath = node.attrib.get('registrationpath', None)

                    if registrationpath is None:
                        registrationpath = os.path.join(
                            self._configuration.application_state_path, app_id)

                    start_action = node.find('Actions/Action[@id="start"]')

                    if start_action is None:
                        logging.warn("No Start Action Found for {0}"
                                     .format(registrationpath))
                        continue

                    dependencies = self._parse_dependencies(start_action)

                    data = {
                        "configuration_path": registrationpath,
                        "dependencies": dependencies,
                        "downstream": list()
                    }

                    result.update({registrationpath: data})

            except Exception:
                logging.exception('An unhandled exception occurred')

        else:
            logging.warn("config path does not exist: {0}".format(path))

    def _parse_dependencies(self, action):
        """
        Parse dependencies out of XML
        :type action: xml.etree.ElementTree.Element
        :rtype: list
        """
        # TODO: rename 'path' when it really isn't a path. this is a hack...
        # prev_was_not keeps track of whether the outer class was 'not'
        dependencies = []
        prev_was_not = False
        for predicate in action.iter('Predicate'):
            pred_type = predicate.get('type').lower()
            pred_path = predicate.get('path', None)
            if pred_type == PredicateType.ZOOKEEPERHASCHILDREN:
                dependencies.append({'type': pred_type,
                                     'path': pred_path})
                prev_was_not = False
            if pred_type == PredicateType.ZOOKEEPERHASGRANDCHILDREN:
                dependencies.append({'type': pred_type,
                                     'path': pred_path})
                prev_was_not = False
            if pred_type == PredicateType.ZOOKEEPERGOODUNTILTIME:
                if len(pred_path.split('gut/')) > 1:
                    dependencies.append(
                        {'type': pred_type,
                         'path': ("I should be up between: {0}"
                                  .format(pred_path.split("gut/")[1]))})
                else:
                    logging.debug('Invalid GUT path: {0}'.format(pred_path))
                prev_was_not = False
            if pred_type == PredicateType.HOLIDAY:
                dependencies.append(
                    {'type': pred_type,
                     'path': ("Does NOT run on holidays" if prev_was_not
                              else "Runs on holidays")})
                prev_was_not = False
            if pred_type == PredicateType.WEEKEND:
                dependencies.append(
                    {'type': pred_type,
                     'path': ("Does NOT run on weekends" if prev_was_not
                              else "Runs on weekends")})
                prev_was_not = False
            if pred_type == PredicateType.TIME:
                start = predicate.get('start', None)
                stop = predicate.get('stop', None)
                msg = 'I should be up '
                if start is not None:
                    msg += 'after: {0} '.format(start)
                if stop is not None:
                    msg += 'until: {0}'.format(stop)
                if not start and not stop:
                    msg += '?'
                dependencies.append({'type': pred_type, 'path': msg})

            if pred_type == PredicateType.NOT:
                prev_was_not = True

        return dependencies

    def _recalc_downstream_dependencies(self):
        """
        Loop over existing cache and link upstream with downstream elements
        """
        # clear existing downstream
        for data in self._cache.application_dependencies.itervalues():
            downstream = data.get('downstream')
            del downstream[:]

        dep_copy = self._cache.application_dependencies.copy()
        for path, data in dep_copy.iteritems():
            for key, value in self._cache.application_dependencies.iteritems():
                for dep in data.get('dependencies'):
                    if dep['type'] == PredicateType.ZOOKEEPERHASGRANDCHILDREN:
                        if key.startswith(dep['path']):
                            value.get('downstream').append(path)
                    elif dep['type'] == PredicateType.ZOOKEEPERHASCHILDREN:
                        if dep['path'] == key:
                            value.get('downstream').append(path)

    def _on_update(self, event):
        """
        Callback to send updates via websocket on application state changes.
        :type event: kazoo.protocol.states.WatchedEvent
        """
        self._on_update_path(event.path)

    def _on_update_path(self, path):
        try:
            message = ApplicationDependenciesMessage()

            self._walk(path, message)
            self._recalc_downstream_dependencies()

            self._cache.update(message.application_dependencies)

            self._message_throttle.add_message(message)

            self._time_estimate_cache.update_dependencies(
                self._cache.application_dependencies)

        except Exception:
            logging.exception('An unhandled Exception has occurred')