def _create_dep_message(self, *args):
"""
:rtype: ApplicationDependenciesMessage
"""
mes = ApplicationDependenciesMessage()
for path in args:
data = {
path: {"configuration_path": path, "dependencies": list()}
}
mes.update(data)
return mes
def __init__(self, configuration, zoo_keeper, web_socket_clients,
time_estimate_cache):
"""
:type configuration: zoom.config.configuration.Configuration
:type zoo_keeper: kazoo.client.KazooClient
:type web_socket_clients: list
"""
self._cache = ApplicationDependenciesMessage()
self._configuration = configuration
self._zoo_keeper = zoo_keeper
self._web_socket_clients = web_socket_clients
self._time_estimate_cache = time_estimate_cache
self._message_throttle = MessageThrottle(configuration,
web_socket_clients)
def __init__(self, configuration, zoo_keeper, web_socket_clients,
time_estimate_cache):
"""
:type configuration: zoom.config.configuration.Configuration
:type zoo_keeper: zoom.www.zoo_keeper.ZooKeeper
:type web_socket_clients: list
"""
self._cache = ApplicationDependenciesMessage()
self._configuration = configuration
self._zoo_keeper = zoo_keeper
self._web_socket_clients = web_socket_clients
self._time_estimate_cache = time_estimate_cache
self._message_throttle = MessageThrottle(configuration,
web_socket_clients)
def _on_update_path(self, path):
try:
message = ApplicationDependenciesMessage()
self._walk(path, message)
self._cache.update(message.application_dependencies)
self._recalc_downstream_dependencies()
self._message_throttle.add_message(message)
self._time_estimate_cache.update_dependencies(
self._cache.application_dependencies)
except Exception:
logging.exception('An unhandled Exception has occurred for path: '
'{0}'.format(path))
def test_update(self):
path = '/foo'
mes = ApplicationDependenciesMessage()
d1 = {"configuration_path": path, "dependencies": list()}
# test dict gets update
data = {path: d1}
mes.update(data)
self.assertEqual(mes.application_dependencies, data)
# test data change for same key
d2 = d1.copy()
d2['dependencies'] = [1, 2, 3]
data = {path: d2}
mes.update(data)
self.assertEqual(mes.application_dependencies.get(path), d2)
class ApplicationDependencyCache(object):
def __init__(self, configuration, zoo_keeper, web_socket_clients,
time_estimate_cache):
"""
:type configuration: zoom.config.configuration.Configuration
:type zoo_keeper: kazoo.client.KazooClient
:type web_socket_clients: list
"""
self._cache = ApplicationDependenciesMessage()
self._configuration = configuration
self._zoo_keeper = zoo_keeper
self._web_socket_clients = web_socket_clients
self._time_estimate_cache = time_estimate_cache
self._message_throttle = MessageThrottle(configuration,
web_socket_clients)
def start(self):
self._message_throttle.start()
def stop(self):
self._message_throttle.stop()
def load(self):
"""
:rtype: ApplicationDependenciesMessage
"""
try:
if not self._cache.application_dependencies:
self._load()
return self._cache
except Exception:
logging.exception('An unhandled Exception has occurred')
def reload(self):
"""
Clear cache, and re-walk agent config path.
"""
self._cache.clear()
logging.info("Application dependency cache cleared")
self._on_update_path(self._configuration.agent_configuration_path)
@TimeThis(__file__)
def _load(self):
"""
Walk full agent config path to get data. Load self._cache object
"""
self._cache.clear()
self._walk(self._configuration.agent_configuration_path, self._cache)
logging.info(
"Application dependency cache loaded from ZooKeeper {0}".format(
self._configuration.agent_configuration_path))
self._recalc_downstream_dependencies()
self._time_estimate_cache.update_dependencies(
self._cache.application_dependencies)
@connected_with_return(None)
def _walk(self, path, result):
"""
:type path: str
:type result: ApplicationDependenciesMessage
"""
try:
children = self._zoo_keeper.get_children(path,
watch=self._on_update)
if children:
for child in children:
self._walk(zk_path_join(path, child), result)
else:
self._get_application_dependency(path, result)
except NoNodeError:
logging.debug('Node at {0} no longer exists.'.format(path))
def _get_application_dependency(self, path, result):
"""
Load result object with application dependencies
:type path: str
:type result: ApplicationDependenciesMessage
"""
if self._zoo_keeper.exists(path):
data, stat = self._zoo_keeper.get(path, watch=self._on_update)
if not data:
return
try:
root = ElementTree.fromstring(data)
for node in root.findall('Automation/Component'):
app_id = node.attrib.get('id')
registrationpath = node.attrib.get('registrationpath',
None)
if registrationpath is None:
registrationpath = zk_path_join(
self._configuration.application_state_path, app_id)
start_action = node.find('Actions/Action[@id="start"]')
if start_action is None:
logging.warn("No Start Action Found for {0}".format(
registrationpath))
dependencies = list()
else:
dependencies = self._parse_dependencies(start_action)
data = {
"configuration_path": registrationpath,
"dependencies": dependencies,
"downstream": list()
}
result.update({registrationpath: data})
except Exception:
logging.exception('An unhandled exception occurred')
else:
logging.warn("config path does not exist: {0}".format(path))
def _parse_dependencies(self, action):
"""
Parse dependencies out of XML
:type action: xml.etree.ElementTree.Element
:rtype: list
"""
# TODO: rename 'path' when it really isn't a path. this is a hack...
# prev_was_not keeps track of whether the outer class was 'not'
dependencies = []
prev_was_not = False
for predicate in action.iter('Predicate'):
pred_type = predicate.get('type').lower()
pred_path = predicate.get('path', None)
# pred_oper = predicate.get('operational', False)
pred_oper = bool(
verify_attribute(predicate, 'operational', none_allowed=True))
if pred_type == PredicateType.ZOOKEEPERHASCHILDREN:
dependencies.append({
'type': pred_type,
'path': pred_path,
'operational': pred_oper
})
prev_was_not = False
elif pred_type == PredicateType.ZOOKEEPERHASGRANDCHILDREN:
dependencies.append({
'type': pred_type,
'path': pred_path,
'operational': pred_oper
})
prev_was_not = False
elif pred_type == PredicateType.ZOOKEEPERGOODUNTILTIME:
if len(pred_path.split('gut/')) > 1:
dependencies.append({
'type':
pred_type,
'path': ("I should be up between: {0}".format(
pred_path.split("gut/")[1])),
'operational':
pred_oper
})
else:
logging.debug('Invalid GUT path: {0}'.format(pred_path))
prev_was_not = False
elif pred_type == PredicateType.HOLIDAY:
dependencies.append({
'type':
pred_type,
'path': ("Does NOT run on holidays"
if prev_was_not else "Runs on holidays"),
'operational':
pred_oper
})
prev_was_not = False
elif pred_type == PredicateType.WEEKEND:
dependencies.append({
'type':
pred_type,
'path': ("Does NOT run on weekends"
if prev_was_not else "Runs on weekends"),
'operational':
pred_oper
})
prev_was_not = False
elif pred_type == PredicateType.TIMEWINDOW:
begin = predicate.get('begin', None)
end = predicate.get('end', None)
weekdays = predicate.get('weekdays', None)
msg = 'I should be up '
if begin is not None:
msg += 'after: {0} '.format(begin)
if end is not None:
msg += 'until: {0}'.format(end)
# only send dependency if there is something to send
if begin is not None or end is not None:
dependencies.append({
'type': pred_type,
'path': msg,
'operational': pred_oper
})
# pretend this is a weekend predicate for convenience
if weekdays is not None:
day_range = TimeWindow.parse_range(weekdays)
if Weekdays.SATURDAY in day_range or Weekdays.SUNDAY in day_range:
wk_msg = 'Runs on weekends'
else:
wk_msg = 'Does NOT run on weekends'
dependencies.append({
'type': PredicateType.WEEKEND,
'path': wk_msg,
'operational': pred_oper
})
elif pred_type == PredicateType.NOT:
prev_was_not = True
return dependencies
@TimeThis(__file__)
def _recalc_downstream_dependencies(self, tries=0):
"""
Loop over existing cache and link upstream with downstream elements
"""
# clear existing downstream
try:
for data in self._cache.application_dependencies.itervalues():
downstream = data.get('downstream')
del downstream[:]
dep_copy = self._cache.application_dependencies.copy()
for path, data in dep_copy.iteritems():
for key, value in self._cache.application_dependencies.iteritems(
):
for dep in data.get('dependencies'):
if dep['type'] == PredicateType.ZOOKEEPERHASGRANDCHILDREN:
if key.startswith(dep['path']):
value.get('downstream').append(path)
elif dep['type'] == PredicateType.ZOOKEEPERHASCHILDREN:
if dep['path'] == key:
value.get('downstream').append(path)
except RuntimeError:
time.sleep(1)
tries += 1
if tries < 3:
self._recalc_downstream_dependencies(tries=tries)
def _on_update(self, event):
"""
Callback to send updates via websocket on application state changes.
:type event: kazoo.protocol.states.WatchedEvent
"""
self._on_update_path(event.path)
def _on_update_path(self, path):
try:
message = ApplicationDependenciesMessage()
self._walk(path, message)
self._cache.update(message.application_dependencies)
self._recalc_downstream_dependencies()
self._message_throttle.add_message(message)
self._time_estimate_cache.update_dependencies(
self._cache.application_dependencies)
except Exception:
logging.exception('An unhandled Exception has occurred for path: '
'{0}'.format(path))
class DataStore(object):
def __init__(self, configuration, zoo_keeper, task_server):
"""
:type configuration: zoom.config.configuration.Configuration
:type zoo_keeper: kazoo.client.KazooClient
:type task_server: zoom.www.entities.task_server.TaskServer
"""
self._configuration = configuration
self._zoo_keeper = zoo_keeper
self._task_server = task_server
self._alert_exceptions = list()
self._pd = \
PagerDuty(self._configuration.pagerduty_subdomain,
self._configuration.pagerduty_api_token,
self._configuration.pagerduty_default_svc_key,
alert_footer=self._configuration.pagerduty_alert_footer)
self._alert_manager = AlertManager(
configuration.alert_path, configuration.override_node,
configuration.application_state_path, zoo_keeper, self._pd,
self._alert_exceptions)
self._web_socket_clients = list()
self._time_estimate_cache = TimeEstimateCache(self._configuration,
self._web_socket_clients)
self._application_dependency_cache = \
ApplicationDependencyCache(self._configuration,
self._zoo_keeper,
self._web_socket_clients,
self._time_estimate_cache)
self._application_state_cache = \
ApplicationStateCache(self._configuration,
self._zoo_keeper,
self._web_socket_clients,
self._time_estimate_cache)
self._global_cache = GlobalCache(self._configuration, self._zoo_keeper,
self._web_socket_clients)
self._pd_svc_list_cache = {}
def start(self):
logging.info('Starting data store.')
self._global_cache.start()
self._application_state_cache.start()
self._application_dependency_cache.start()
self._time_estimate_cache.start()
self._alert_manager.start()
def stop(self):
logging.info('Stopping data store.')
self._global_cache.stop()
self._application_state_cache.stop()
self._application_dependency_cache.stop()
self._time_estimate_cache.stop()
self._alert_manager.stop()
@connected_with_return(ApplicationStatesMessage())
def load_application_state_cache(self):
"""
:rtype: zoom.messages.application_states.ApplicationStatesMessage
"""
logging.info('Loading application states.')
return self._application_state_cache.load()
@connected_with_return(ApplicationDependenciesMessage())
def load_application_dependency_cache(self):
"""
:rtype: zoom.messages.application_dependencies.ApplicationDependenciesMessage
"""
logging.info('Loading application dependencies.')
return self._application_dependency_cache.load()
@connected_with_return(TimeEstimateMessage())
def load_time_estimate_cache(self):
"""
:rtype: zoom.messages.timing_estimate.TimeEstimateMessage
"""
return self._time_estimate_cache.load()
def get_start_time(self, path):
"""
:rtype: dict
"""
return self._time_estimate_cache.get_graphite_data(path)
@connected_with_return(GlobalModeMessage('{"mode":"Unknown"}'))
def get_global_mode(self):
"""
:rtype: zoom.messages.global_mode_message.GlobalModeMessage
"""
logging.info('Loading global mode.')
return self._global_cache.get_mode()
def reload(self):
"""
Clear all cache objects and send reloaded data as updates.
"""
# restart client to destroy any existing watches
# self._zoo_keeper.restart()
logging.info('Reloading all cache types.')
self._task_server.clear_all_tasks()
self._global_cache.on_update()
self._application_state_cache.reload()
self._application_dependency_cache.reload()
self._time_estimate_cache.reload()
self._alert_manager.start()
self._pd_svc_list_cache = self._pd.get_service_dict()
return {'cache_clear': 'okay'}
def load(self):
"""
Clear all cache objects and send reloaded data as updates.
"""
logging.info('Loading all cache types.')
self._global_cache.on_update()
self._application_state_cache.load()
self._application_dependency_cache.load()
self._time_estimate_cache.load()
self._pd_svc_list_cache = self._pd.get_service_dict()
return {'cache_load': 'okay'}
@property
def pd_client(self):
"""
:rtype: zoom.common.pagerduty.PagerDuty
"""
return self._pd
@property
def web_socket_clients(self):
"""
:rtype: list
"""
return self._web_socket_clients
@property
def application_state_cache(self):
"""
:rtype: zoom.www.cache.application_state_cache.ApplicationStateCache
"""
return self._application_state_cache
@property
def alert_exceptions(self):
"""
:rtype: list
"""
return self._alert_exceptions
@property
def pagerduty_services(self):
"""
:rtype: dict
"""
return self._pd_svc_list_cache
class ApplicationDependencyCache(object):
def __init__(self, configuration, zoo_keeper, web_socket_clients,
time_estimate_cache):
"""
:type configuration: zoom.config.configuration.Configuration
:type zoo_keeper: zoom.www.zoo_keeper.ZooKeeper
:type web_socket_clients: list
"""
self._cache = ApplicationDependenciesMessage()
self._configuration = configuration
self._zoo_keeper = zoo_keeper
self._web_socket_clients = web_socket_clients
self._time_estimate_cache = time_estimate_cache
self._message_throttle = MessageThrottle(configuration,
web_socket_clients)
def start(self):
self._message_throttle.start()
def stop(self):
self._message_throttle.stop()
def load(self):
"""
:rtype: ApplicationDependenciesMessage
"""
try:
if not self._cache.application_dependencies:
self._load()
return self._cache
except Exception:
logging.exception('An unhandled Exception has occurred')
def reload(self):
"""
Clear cache, and re-walk agent config path.
"""
self._cache.clear()
logging.info("Application dependency cache cleared")
self._on_update_path(self._configuration.agent_configuration_path)
def _load(self):
"""
Walk full agent config path to get data. Load self._cache object
"""
self._cache.clear()
self._walk(self._configuration.agent_configuration_path, self._cache)
logging.info("Application dependency cache loaded from ZooKeeper {0}"
.format(self._configuration.agent_configuration_path))
self._recalc_downstream_dependencies()
self._time_estimate_cache.update_dependencies(
self._cache.application_dependencies)
@connected_with_return(None)
def _walk(self, path, result):
"""
:type path: str
:type result: ApplicationDependenciesMessage
"""
try:
children = self._zoo_keeper.get_children(path,
watch=self._on_update)
if children:
for child in children:
self._walk(os.path.join(path, child), result)
else:
self._get_application_dependency(path, result)
except NoNodeError:
logging.debug('Node at {0} no longer exists.'.format(path))
def _get_application_dependency(self, path, result):
"""
Load result object with application dependencies
:type path: str
:type result: ApplicationDependenciesMessage
"""
if self._zoo_keeper.exists(path):
data, stat = self._zoo_keeper.get(path, watch=self._on_update)
if not data:
return
try:
root = ElementTree.fromstring(data)
for node in root.findall('Automation/Component'):
app_id = node.attrib.get('id')
registrationpath = node.attrib.get('registrationpath', None)
if registrationpath is None:
registrationpath = os.path.join(
self._configuration.application_state_path, app_id)
start_action = node.find('Actions/Action[@id="start"]')
if start_action is None:
logging.warn("No Start Action Found for {0}"
.format(registrationpath))
continue
dependencies = self._parse_dependencies(start_action)
data = {
"configuration_path": registrationpath,
"dependencies": dependencies,
"downstream": list()
}
result.update({registrationpath: data})
except Exception:
logging.exception('An unhandled exception occurred')
else:
logging.warn("config path does not exist: {0}".format(path))
def _parse_dependencies(self, action):
"""
Parse dependencies out of XML
:type action: xml.etree.ElementTree.Element
:rtype: list
"""
# TODO: rename 'path' when it really isn't a path. this is a hack...
# prev_was_not keeps track of whether the outer class was 'not'
dependencies = []
prev_was_not = False
for predicate in action.iter('Predicate'):
pred_type = predicate.get('type').lower()
pred_path = predicate.get('path', None)
if pred_type == PredicateType.ZOOKEEPERHASCHILDREN:
dependencies.append({'type': pred_type,
'path': pred_path})
prev_was_not = False
if pred_type == PredicateType.ZOOKEEPERHASGRANDCHILDREN:
dependencies.append({'type': pred_type,
'path': pred_path})
prev_was_not = False
if pred_type == PredicateType.ZOOKEEPERGOODUNTILTIME:
if len(pred_path.split('gut/')) > 1:
dependencies.append(
{'type': pred_type,
'path': ("I should be up between: {0}"
.format(pred_path.split("gut/")[1]))})
else:
logging.debug('Invalid GUT path: {0}'.format(pred_path))
prev_was_not = False
if pred_type == PredicateType.HOLIDAY:
dependencies.append(
{'type': pred_type,
'path': ("Does NOT run on holidays" if prev_was_not
else "Runs on holidays")})
prev_was_not = False
if pred_type == PredicateType.WEEKEND:
dependencies.append(
{'type': pred_type,
'path': ("Does NOT run on weekends" if prev_was_not
else "Runs on weekends")})
prev_was_not = False
if pred_type == PredicateType.TIME:
start = predicate.get('start', None)
stop = predicate.get('stop', None)
msg = 'I should be up '
if start is not None:
msg += 'after: {0} '.format(start)
if stop is not None:
msg += 'until: {0}'.format(stop)
if not start and not stop:
msg += '?'
dependencies.append({'type': pred_type, 'path': msg})
if pred_type == PredicateType.NOT:
prev_was_not = True
return dependencies
def _recalc_downstream_dependencies(self):
"""
Loop over existing cache and link upstream with downstream elements
"""
# clear existing downstream
for data in self._cache.application_dependencies.itervalues():
downstream = data.get('downstream')
del downstream[:]
dep_copy = self._cache.application_dependencies.copy()
for path, data in dep_copy.iteritems():
for key, value in self._cache.application_dependencies.iteritems():
for dep in data.get('dependencies'):
if dep['type'] == PredicateType.ZOOKEEPERHASGRANDCHILDREN:
if key.startswith(dep['path']):
value.get('downstream').append(path)
elif dep['type'] == PredicateType.ZOOKEEPERHASCHILDREN:
if dep['path'] == key:
value.get('downstream').append(path)
def _on_update(self, event):
"""
Callback to send updates via websocket on application state changes.
:type event: kazoo.protocol.states.WatchedEvent
"""
self._on_update_path(event.path)
def _on_update_path(self, path):
try:
message = ApplicationDependenciesMessage()
self._walk(path, message)
self._recalc_downstream_dependencies()
self._cache.update(message.application_dependencies)
self._message_throttle.add_message(message)
self._time_estimate_cache.update_dependencies(
self._cache.application_dependencies)
except Exception:
logging.exception('An unhandled Exception has occurred')
