def recover_prometheus(self, provider, cluster):
if provider["type"] == "master":
if not self.container_stopped("prometheus"):
self.logger.info("prometheus container is already running")
return
self.logger.warn("prometheus container is not running")
self.logger.info("restarting prometheus container")
self.docker.restart("prometheus")
addr, prefixlen = get_prometheus_cidr(cluster["weave_ip_network"])
sh.weave("attach", "{}/{}".format(addr, prefixlen), "prometheus")
def recover_weave(self, provider, cluster):
try:
if not self.container_stopped("weave"):
self.logger.info("weave container is already running")
return
except docker.errors.APIError as exc:
err_code = exc.response.status_code
if err_code == 404:
self.logger.warn(exc)
else:
raise
self.logger.warn("weave container is not running")
self.logger.info("restarting weave container")
passwd = ""
if self.encrypted:
passwd = decrypt_text(cluster["admin_pw"], cluster["passkey"])
if provider["type"] == "master":
sh.weave(
"launch-router",
"--password", passwd,
"--dns-domain", "gluu.local",
"--ipalloc-range", cluster["weave_ip_network"],
"--ipalloc-default-subnet", cluster["weave_ip_network"],
)
else:
with open("/etc/salt/minion") as fp:
config = fp.read()
opts = yaml.safe_load(config)
sh.weave(
"launch-router",
"--password", passwd,
"--dns-domain", "gluu.local",
"--ipalloc-range", cluster["weave_ip_network"],
"--ipalloc-default-subnet", cluster["weave_ip_network"],
opts["master"],
)
addr, prefixlen = get_exposed_cidr(cluster["weave_ip_network"])
sh.weave("expose", "{}/{}".format(addr, prefixlen))
def recover_nodes(self, provider, cluster):
_nodes = self.db.search_from_table(
"nodes",
(self.db.where("provider_id") == provider["id"])
& (self.db.where("state") == STATE_SUCCESS)
)
# attach the recovery priority
success_nodes = [format_node(node) for node in _nodes]
# disabled nodes must be recovered so we can enable again when
# expired license is updated
_nodes = self.db.search_from_table(
"nodes",
(self.db.where("provider_id") == provider["id"])
& (self.db.where("state") == STATE_DISABLED)
)
# attach the recovery priority
disabled_nodes = [format_node(node) for node in _nodes]
# sort nodes by its recovery_priority property
# so we will have a fully recovered nodes
nodes = sorted(success_nodes + disabled_nodes,
key=lambda node: node["recovery_priority"])
for node in nodes:
if not self.container_stopped(node["id"]):
self.logger.info("{} node {} is already running".format(
node["type"], node["id"]
))
# if weave is relaunched by another tool, DNS entries
# might not be restored, hence we're readding the entries
sh.weave("dns-add", node["id"], "-h", node["domain_name"])
if node["type"] == "ldap":
sh.weave("dns-add", node["id"], "-h", "ldap.gluu.local")
if node["type"] == "nginx":
sh.weave("dns-add", node["id"], "-h", cluster["ox_cluster_hostname"]) # noqa
continue
self.logger.warn("{} node {} is not running; restarting ..".format(
node["type"], node["id"]
))
self.docker.restart(node["id"])
if node["state"] == STATE_SUCCESS:
cidr = "{}/{}".format(node["weave_ip"],
node["weave_prefixlen"])
self.logger.info("attaching weave IP {}".format(cidr))
sh.weave("attach", "{}".format(cidr), node["id"])
self.logger.info("adding {} to local "
"DNS server".format(node["domain_name"]))
sh.weave("dns-add", node["id"], "-h", node["domain_name"])
if node["type"] == "ldap":
self.logger.info("adding ldap.gluu.local to "
"local DNS server")
sh.weave("dns-add", node["id"], "-h", "ldap.gluu.local")
if node["type"] == "nginx":
sh.weave("dns-add", node["id"], "-h", cluster["ox_cluster_hostname"]) # noqa
self.setup_node(node, provider, cluster)