def run(self, errors, error_level):
errors = json.loads(errors, encoding="utf-8")
if "name" not in errors:
raise Exception(
"Error: Instance name not provided in errors. Can not continue!"
)
instance = errors["name"]
vtm = Vtm(self.config, self.logger, instance)
bsd = Bsd(self.config, self.logger)
status = bsd.get_status(instance)[0]
nodes = errors["traffic_health"]["failed_nodes"]
failedPools = {
pool: [node["node"] for node in nodes for pool in node["pools"]]
for node in nodes for pool in node["pools"]
}
for pool in failedPools.keys():
nodes = vtm.get_pool_nodes(pool)
if set(nodes["active"]).issubset(failedPools[pool]):
self.logger.debug("Pool Dead")
for vs in status["traffic_health"]["virtual_servers"]:
if vs["pool"] == pool:
self.logger.debug(
"Putting VS: {} into maintenance.".format(
vs["name"]))
vtm.enable_maintenance(vs["name"], "maintenance")
else:
self.logger.debug("Pool not dead")
def run(self, errors, error_level):
errors = json.loads(errors, encoding="utf-8")
if "name" not in errors:
raise Exception("Error: Instance name not provided in errors. Can not continue!")
instance = errors["name"]
vtm = Vtm(self.config, self.logger, instance)
bsd = Bsd(self.config, self.logger)
status = bsd.getStatus(instance)[0]
nodes = errors["traffic_health"]["failed_nodes"]
failedPools = {
pool: [node["node"] for node in nodes for pool in node["pools"]] for node in nodes for pool in node["pools"]
}
for pool in failedPools.keys():
nodes = vtm.getPoolNodes(pool)
if set(nodes["active"]).issubset(failedPools[pool]):
self.logger.debug("Pool Dead")
for vs in status["traffic_health"]["virtual_servers"]:
if vs["pool"] == pool:
self.logger.debug("Putting VS: {} into maintenance.".format(vs["name"]))
vtm.enableMaintenance(vs["name"], "maintenance")
else:
self.logger.debug("Pool not dead")
def run(self, vtm, stringify):
bsd = Bsd(self.config, self.logger)
result = bsd.getStatus(vtm, stringify)
return result
def run(self, vtm, bw):
bsd = Bsd(self.config, self.logger)
bsd.setBandwidth(vtm, bw)
class brcdSdSensor(PollingSensor):
def __init__(self, sensor_service, config=None, poll_interval=5):
super(brcdSdSensor, self).__init__(sensor_service=sensor_service,
config=config,
poll_interval=poll_interval)
self._logger = self.sensor_service.get_logger(
name=self.__class__.__name__)
self._stop = False
self._bsd = None
def setup(self):
self._bsd = Bsd(self._config, self._logger)
self._last_errors = None
def poll(self):
try:
errors = self._bsd.get_errors()
last_errors = self._get_last_errors()
self._set_last_errors(errors)
self._process_changes(errors, last_errors)
if last_errors and not errors:
payload = {
"status": "all_clear",
"errors": "ALL CLEAR",
"error_level": "na"
}
self.sensor_service.dispatch(trigger="vadc.bsd_failure_event",
payload=payload)
except Exception as e:
payload = {
"status":
"sensor_fail",
"errors":
"BSD Sensor: {}: {}".format(self._config["brcd_sd_host"], e),
"error_level":
"na"
}
self.sensor_service.dispatch(trigger="vadc.bsd_failure_event",
payload=payload)
def _process_changes(self, errors, last_errors):
for instance in errors:
if "traffic_health" in errors[instance]:
error_level = errors[instance]["traffic_health"]["error_level"]
else:
error_level = "unknown"
if instance in last_errors:
if errors[instance] != last_errors[instance]:
# This has changed
payload = {
"status": "updated",
"error_level": error_level,
"errors": json.dumps(errors[instance],
encoding="utf-8")
}
self.sensor_service.dispatch(
trigger="vadc.bsd_failure_event", payload=payload)
else:
# No change
pass
else:
# New error
payload = {
"status": "new",
"error_level": error_level,
"errors": json.dumps(errors[instance], encoding="utf-8")
}
self.sensor_service.dispatch(trigger="vadc.bsd_failure_event",
payload=payload)
for instance in last_errors:
if instance not in errors:
# Recovered
payload = {
"status": "resolved",
"error_level": "ok",
"errors": json.dumps(last_errors[instance],
encoding="utf-8")
}
self.sensor_service.dispatch(trigger="vadc.bsd_failure_event",
payload=payload)
def _get_last_errors(self):
if not self._last_errors and hasattr(self._sensor_service,
'get_value'):
last_errors = self._sensor_service.get_value(name='last_errors')
if last_errors is not None:
self._last_errors = json.loads(last_errors, encoding="utf-8")
else:
self._last_errors = {}
return self._last_errors
def _set_last_errors(self, last_errors):
self._last_errors = last_errors
if hasattr(self._sensor_service, 'set_value'):
self._sensor_service.set_value(name='last_errors',
value=json.dumps(last_errors,
encoding="utf-8"))
def cleanup(self):
# This is called when the st2 system goes down. You can perform cleanup operations like
# closing the connections to external system here.
pass
def add_trigger(self, trigger):
# This method is called when trigger is created
pass
def update_trigger(self, trigger):
# This method is called when trigger is updated
pass
def remove_trigger(self, trigger):
# This method is called when trigger is deleted
pass
def run(self, vtm, stringify):
bsd = Bsd(self.config, self.logger)
result = bsd.get_bandwidth(vtm, stringify)
return (True, result)
def run(self, vtm, password, address, bw, fp):
bsd = Bsd(self.config, self.logger)
result = bsd.addVtm(vtm, password, address, bw, fp)
return result
def run(self, vtm):
bsd = Bsd(self.config, self.logger)
result = bsd.del_vtm(vtm)
return (True, result)
def run(self, vtm):
bsd = Bsd(self.config, self.logger)
result = bsd.delVtm(vtm)
return result
def run(self, stringify):
bsd = Bsd(self.config, self.logger)
result = bsd.get_errors(stringify)
return (True, result)
def run(self, vtms, cluster):
bsd = Bsd(self.config, self.logger)
result = bsd.get_active_vtm(vtms, cluster)
if result is None:
return (False, result)
return (True, result)
class brcdBwSensor(PollingSensor):
def __init__(self, sensor_service, config=None, poll_interval=5):
super(brcdBwSensor, self).__init__(sensor_service=sensor_service, config=config,
poll_interval=poll_interval)
self._logger = self.sensor_service.get_logger(name=self.__class__.__name__)
self._stop = False
self._bw_tracker = None
self._bsd = None
self._manage = None
self._minimum = 10
self._headroom = 10
self._roundup = 10
self._track = 10
self._warn = 10
def setup(self):
self._bsd = Bsd(self._config, self._logger)
self._bw_tracker = None
def poll(self):
self._get_configs()
bandwidth = self._bsd.getBandwidth()
bw_tracker = self._get_bw_tracker()
if self._manage is not None:
self._manage_bandwidth(bw_tracker, bandwidth)
else:
bw_tracker = {}
self._monitor_bandwidth(bandwidth)
self._set_bw_tracker(bw_tracker)
def _get_configs(self):
self._logger.debug(self._config)
if "brcd_bw_minimum" in self._config and self._config["brcd_bw_minimum"] is not None:
self._minimum = int(self._config["brcd_bw_minimum"])
if "brcd_bw_roundup" in self._config and self._config["brcd_bw_roundup"] is not None:
self._roundup = int(self._config["brcd_bw_roundup"])
if "brcd_bw_headroom" in self._config and self._config["brcd_bw_headroom"] is not None:
self._headroom = int(self._config["brcd_bw_headroom"])
if "brcd_bw_track" in self._config and self._config["brcd_bw_track"] is not None:
self._track = int(self._config["brcd_bw_track"])
if "brcd_bw_warn" in self._config and self._config["brcd_bw_warn"] is not None:
self._warn = int(self._config["brcd_bw_warn"])
if "brcd_bw_manage" in self._config and self._config["brcd_bw_manage"] is not None:
manage = str(self._config["brcd_bw_manage"])
if manage.lower() == "all":
self._manage = ["__ALL__"]
else:
self._manage = manage.split(',')
def _manage_bandwidth(self, bw_tracker, bandwidth):
for instance in bandwidth.keys():
instData = bandwidth[instance]
tag = instData["tag"]
if "__ALL__" in self._manage or instance in self._manage or tag in self._manage:
tracked = instData["current"]
tracked += self._roundup - (tracked % self._roundup)
if tracked < self._minimum:
tracked = self._minimum
if instance not in bw_tracker:
bw_tracker[instance] = instData
bw_tracker[instance]["tracking"] = []
while len(bw_tracker[instance]["tracking"]) >= self._track:
bw_tracker[instance]["tracking"].pop(0)
bw_tracker[instance]["tracking"].append(tracked)
bw_tracker[instance]["current"] = instData["current"]
bw_tracker[instance]["peak"] = instData["peak"]
self._issue_update(instance, tracked, instData, bw_tracker, bandwidth)
else:
if instance in bw_tracker:
bw_tracker.pop(instance)
def _issue_update(self, instance, tracked, instData, bw_tracker, bandwidth):
average = sum(bw_tracker[instance]["tracking"]) / len(bw_tracker[instance]["tracking"])
assign = average if tracked < average else tracked
assign = int(assign + self._headroom)
if assign != instData["assigned"]:
bw_tracker[instance]["assigned"] = assign
payload = {"action": "update", "instance": instance, "tag": instData["tag"],
"bandwidth": assign, "current": instData["current"], "average": average}
self.sensor_service.dispatch(trigger="vadc.bsd_bandwidth_event", payload=payload)
def _monitor_bandwidth(self, bandwidth):
for instance in bandwidth.keys():
instData = bandwidth[instance]
tag = instData["tag"]
if instData["assigned"] - instData["current"] - self._warn <= 0:
payload = {"action": "alert", "instance": instance, "tag": tag,
"bandwidth": instData["assigned"], "current": instData["current"]}
self.sensor_service.dispatch(trigger="vadc.bsd_bandwidth_event", payload=payload)
def _get_bw_tracker(self):
if not self._bw_tracker and hasattr(self._sensor_service, 'get_value'):
bw_tracker = self._sensor_service.get_value(name='bw_tracker')
if bw_tracker is not None:
self._bw_tracker = json.loads(bw_tracker, encoding="utf-8")
else:
self._bw_tracker = {}
return self._bw_tracker
def _set_bw_tracker(self, bw_tracker):
self._bw_tracker = bw_tracker
if hasattr(self._sensor_service, 'set_value'):
self._sensor_service.set_value(name='bw_tracker', value=json.dumps(bw_tracker,
encoding="utf-8"))
def cleanup(self):
# This is called when the st2 system goes down. You can perform cleanup operations like
# closing the connections to external system here.
pass
def add_trigger(self, trigger):
# This method is called when trigger is created
pass
def update_trigger(self, trigger):
# This method is called when trigger is updated
pass
def remove_trigger(self, trigger):
# This method is called when trigger is deleted
pass
def setup(self):
self._bsd = Bsd(self._config, self._logger)
self._bw_tracker = None
class brcdSdSensor(PollingSensor):
def __init__(self, sensor_service, config=None, poll_interval=5):
super(brcdSdSensor, self).__init__(sensor_service=sensor_service, config=config,
poll_interval=poll_interval)
self._logger = self.sensor_service.get_logger(name=self.__class__.__name__)
self._stop = False
self._bsd = None
def setup(self):
self._bsd = Bsd(self._config, self._logger)
self._last_errors = None
def poll(self):
try:
errors = self._bsd.getErrors()
last_errors = self._get_last_errors()
self._set_last_errors(errors)
self._process_changes(errors, last_errors)
if last_errors and not errors:
payload = {"status": "all_clear", "errors": "ALL CLEAR", "error_level": "na"}
self.sensor_service.dispatch(trigger="vadc.bsd_failure_event", payload=payload)
except Exception as e:
payload = {"status": "sensor_fail", "errors": "BSD Sensor: {}: {}".format(
self._config["brcd_sd_host"], e), "error_level": "na"}
self.sensor_service.dispatch(trigger="vadc.bsd_failure_event", payload=payload)
def _process_changes(self, errors, last_errors):
for instance in errors:
if "traffic_health" in errors[instance]:
error_level = errors[instance]["traffic_health"]["error_level"]
else:
error_level = "unknown"
if instance in last_errors:
if errors[instance] != last_errors[instance]:
# This has changed
payload = {"status": "updated", "error_level": error_level, "errors":
json.dumps(errors[instance], encoding="utf-8")}
self.sensor_service.dispatch(trigger="vadc.bsd_failure_event", payload=payload)
else:
# No change
pass
else:
# New error
payload = {"status": "new", "error_level": error_level, "errors":
json.dumps(errors[instance], encoding="utf-8")}
self.sensor_service.dispatch(trigger="vadc.bsd_failure_event", payload=payload)
for instance in last_errors:
if instance not in errors:
# Recovered
payload = {"status": "resolved", "error_level": "ok", "errors":
json.dumps(last_errors[instance], encoding="utf-8")}
self.sensor_service.dispatch(trigger="vadc.bsd_failure_event", payload=payload)
def _get_last_errors(self):
if not self._last_errors and hasattr(self._sensor_service, 'get_value'):
last_errors = self._sensor_service.get_value(name='last_errors')
if last_errors is not None:
self._last_errors = json.loads(last_errors, encoding="utf-8")
else:
self._last_errors = {}
return self._last_errors
def _set_last_errors(self, last_errors):
self._last_errors = last_errors
if hasattr(self._sensor_service, 'set_value'):
self._sensor_service.set_value(name='last_errors', value=json.dumps(last_errors,
encoding="utf-8"))
def cleanup(self):
# This is called when the st2 system goes down. You can perform cleanup operations like
# closing the connections to external system here.
pass
def add_trigger(self, trigger):
# This method is called when trigger is created
pass
def update_trigger(self, trigger):
# This method is called when trigger is updated
pass
def remove_trigger(self, trigger):
# This method is called when trigger is deleted
pass
def setup(self):
self._bsd = Bsd(self._config, self._logger)
self._last_errors = None
def setup(self):
self._bsd = Bsd(self._config, self._logger)
self._bw_tracker = None
class brcdBwSensor(PollingSensor):
def __init__(self, sensor_service, config=None, poll_interval=5):
super(brcdBwSensor, self).__init__(sensor_service=sensor_service,
config=config,
poll_interval=poll_interval)
self._logger = self.sensor_service.get_logger(
name=self.__class__.__name__)
self._stop = False
self._bw_tracker = None
self._bsd = None
self._manage = None
self._minimum = 10
self._headroom = 10
self._roundup = 10
self._track = 10
self._warn = 10
def setup(self):
self._bsd = Bsd(self._config, self._logger)
self._bw_tracker = None
def poll(self):
self._get_configs()
bandwidth = self._bsd.get_bandwidth()
bw_tracker = self._get_bw_tracker()
if self._manage is not None:
self._manage_bandwidth(bw_tracker, bandwidth)
else:
bw_tracker = {}
self._monitor_bandwidth(bandwidth)
self._set_bw_tracker(bw_tracker)
def _get_configs(self):
self._logger.debug(self._config)
if "brcd_bw_minimum" in self._config and self._config[
"brcd_bw_minimum"] is not None:
self._minimum = int(self._config["brcd_bw_minimum"])
if "brcd_bw_roundup" in self._config and self._config[
"brcd_bw_roundup"] is not None:
self._roundup = int(self._config["brcd_bw_roundup"])
if "brcd_bw_headroom" in self._config and self._config[
"brcd_bw_headroom"] is not None:
self._headroom = int(self._config["brcd_bw_headroom"])
if "brcd_bw_track" in self._config and self._config[
"brcd_bw_track"] is not None:
self._track = int(self._config["brcd_bw_track"])
if "brcd_bw_warn" in self._config and self._config[
"brcd_bw_warn"] is not None:
self._warn = int(self._config["brcd_bw_warn"])
if "brcd_bw_manage" in self._config and self._config[
"brcd_bw_manage"] is not None:
manage = str(self._config["brcd_bw_manage"])
if manage.lower() == "all":
self._manage = ["__ALL__"]
else:
self._manage = manage.split(',')
def _manage_bandwidth(self, bw_tracker, bandwidth):
for instance in bandwidth.keys():
instData = bandwidth[instance]
tag = instData["tag"]
if "__ALL__" in self._manage or instance in self._manage or tag in self._manage:
tracked = instData["current"]
tracked += self._roundup - (tracked % self._roundup)
if tracked < self._minimum:
tracked = self._minimum
if instance not in bw_tracker:
bw_tracker[instance] = instData
bw_tracker[instance]["tracking"] = []
while len(bw_tracker[instance]["tracking"]) >= self._track:
bw_tracker[instance]["tracking"].pop(0)
bw_tracker[instance]["tracking"].append(tracked)
bw_tracker[instance]["current"] = instData["current"]
bw_tracker[instance]["peak"] = instData["peak"]
self._issue_update(instance, tracked, instData, bw_tracker,
bandwidth)
else:
if instance in bw_tracker:
bw_tracker.pop(instance)
for instance in bw_tracker.keys():
if instance not in bandwidth:
bw_tracker.pop(instance)
def _issue_update(self, instance, tracked, instData, bw_tracker,
bandwidth):
average = sum(bw_tracker[instance]["tracking"]) / len(
bw_tracker[instance]["tracking"])
assign = average if tracked < average else tracked
assign = int(assign + self._headroom)
if assign != instData["assigned"]:
bw_tracker[instance]["assigned"] = assign
payload = {
"action": "update",
"instance": instance,
"tag": instData["tag"],
"bandwidth": assign,
"current": instData["current"],
"average": average
}
self.sensor_service.dispatch(trigger="vadc.bsd_bandwidth_event",
payload=payload)
def _monitor_bandwidth(self, bandwidth):
for instance in bandwidth.keys():
instData = bandwidth[instance]
tag = instData["tag"]
if instData["assigned"] - instData["current"] - self._warn <= 0:
payload = {
"action": "alert",
"instance": instance,
"tag": tag,
"bandwidth": instData["assigned"],
"current": instData["current"]
}
self.sensor_service.dispatch(
trigger="vadc.bsd_bandwidth_event", payload=payload)
def _get_bw_tracker(self):
if not self._bw_tracker and hasattr(self._sensor_service, 'get_value'):
bw_tracker = self._sensor_service.get_value(name='bw_tracker')
if bw_tracker is not None:
self._bw_tracker = json.loads(bw_tracker, encoding="utf-8")
else:
self._bw_tracker = {}
return self._bw_tracker
def _set_bw_tracker(self, bw_tracker):
self._bw_tracker = bw_tracker
if hasattr(self._sensor_service, 'set_value'):
self._sensor_service.set_value(name='bw_tracker',
value=json.dumps(bw_tracker,
encoding="utf-8"))
def cleanup(self):
# This is called when the st2 system goes down. You can perform cleanup operations like
# closing the connections to external system here.
pass
def add_trigger(self, trigger):
# This method is called when trigger is created
pass
def update_trigger(self, trigger):
# This method is called when trigger is updated
pass
def remove_trigger(self, trigger):
# This method is called when trigger is deleted
pass
def run(self, full, deleted, stringify):
bsd = Bsd(self.config, self.logger)
result = bsd.listVtms(full, deleted, stringify)
return result
def run(self, vtm, bw):
bsd = Bsd(self.config, self.logger)
bsd.set_bandwidth(vtm, bw)
def run(self, stringify):
bsd = Bsd(self.config, self.logger)
result = bsd.getErrors(stringify)
return result
def run(self, full, deleted, stringify):
bsd = Bsd(self.config, self.logger)
result = bsd.list_vtms(full, deleted, stringify)
return (True, result)
def run(self, vtm, stringify):
bsd = Bsd(self.config, self.logger)
result = bsd.getStatus(vtm, stringify)
return result
def setup(self):
self._bsd = Bsd(self._config, self._logger)
self._last_errors = None
def run(self, vtm, stringify):
bsd = Bsd(self.config, self.logger)
result = bsd.getBandwidth(vtm, stringify)
return result
def run(self, vtm, password, address, bw, fp):
bsd = Bsd(self.config, self.logger)
result = bsd.addVtm(vtm, password, address, bw, fp)
return result
def run(self, vtm):
bsd = Bsd(self.config, self.logger)
result = bsd.delVtm(vtm)
return result
