def network_check_health(net_id, retries=60, period=5):
"""
Check the chain health. If not healthy, will reset the chain
:param net_id: id of the chain
:param retries: how many retries before thinking not health
:param period: wait between two retries
:return:
"""
net = cluster_handler.get_by_id(net_id)
if not net:
logger.warning("Not find chain {}".format(net_id))
return
if net.get("status") != NETWORK_STATUS_RUNNING: # check running one
return
net_name = net.get("name")
logger.debug("Chain {}/{}: checking health".format(net_name, net_id))
# free or used by user, then check its health
for i in range(retries):
if cluster_handler.refresh_health(net_id): # chain is healthy
return
else:
logger.debug("Health Check {}: cluster {}/{} is unhealthy!".format(
i, net_name, net_id))
time.sleep(period)
logger.warning("Chain {}/{} is unhealthy!".format(net_name, net_id))
# only reset free chains
if cluster_handler.get_by_id(net_id).get("user_id") == "":
logger.info("Timeout....resetting free unhealthy chain {}/{}".format(
net_name, net_id))
cluster_handler.reset_free_one(net_id)
def cluster_info_show(cluster_id):
logger.debug("/ cluster_info/{}?released={} action={}".format(
cluster_id, r.args.get('released', '0'), r.method))
released = (r.args.get('released', '0') != '0')
if not released:
return render_template("cluster_info.html",
item=cluster_handler.get_by_id(cluster_id),
consensus_plugins=CONSENSUS_PLUGINS)
else:
return render_template("cluster_info.html",
item=cluster_handler.get_by_id(
cluster_id, col_name="released"),
consensus_plugins=CONSENSUS_PLUGINS)
def cluster_info_show(cluster_id):
logger.debug("/ cluster_info/{}?released={} action={}".format(
cluster_id, r.args.get('released', '0'), r.method))
released = (r.args.get('released', '0') != '0')
if not released:
return render_template("cluster_info.html",
item=cluster_handler.get_by_id(cluster_id),
consensus_plugins=CONSENSUS_PLUGINS_FABRIC_V1)
else:
return render_template("cluster_info.html",
item=cluster_handler.get_by_id(
cluster_id, col_name="released"),
consensus_plugins=CONSENSUS_PLUGINS_FABRIC_V1)
def chain_check_health(chain_id, retries=3, period=5):
"""
Check the chain health.
:param chain_id: id of the chain
:param retries: how many retries before thinking not health
:param period: wait between two retries
:return:
"""
# if not cluster_handler.check_health(chain_id) \
# and c['user_id'] != SYS_UNHEALTHY:
# cluster_handler.release_cluster(c['id'], record=False)
chain = cluster_handler.get_by_id(chain_id)
if not chain:
logger.warning("Not find chain with id = {}".format(chain_id))
return
chain_user_id = chain.get("user_id")
chain_name = chain.get("name")
logger.debug("Chain {}/{}: checking health".format(chain_name, chain_id))
# we should never process in-processing chains unless deleting one
if chain_user_id.startswith(SYS_USER):
if chain_user_id.startswith(SYS_DELETER): # in system processing, TBD
for i in range(retries):
time.sleep(period)
if cluster_handler.get_by_id(chain_id).get("user_id") != \
chain_user_id:
return
logger.info("Delete in-deleting chain {}/{}".format(
chain_name, chain_id))
cluster_handler.delete(chain_id)
return
# free or used by user, then check its health
for i in range(retries):
if cluster_handler.refresh_health(chain_id): # chain is healthy
return
else:
time.sleep(period)
logger.warning("Chain {}/{} is unhealthy!".format(chain_name, chain_id))
# only reset free chains
if cluster_handler.get_by_id(chain_id).get("user_id") == "":
logger.info("Deleting free unhealthy chain {}/{}".format(
chain_name, chain_id))
# cluster_handler.delete(chain_id)
cluster_handler.reset_free_one(chain_id)
def chain_check_health(chain_id, retries=3, period=5):
"""
Check the chain health.
:param chain_id: id of the chain
:param retries: how many retries before thinking not health
:param period: wait between two retries
:return:
"""
chain = cluster_handler.get_by_id(chain_id)
if not chain:
logger.warning("Not find chain {}".format(chain_id))
return
chain_user_id = chain.get("user_id")
chain_name = chain.get("name")
logger.debug("Chain {}/{}: checking health".format(chain_name, chain_id))
# we should never process in-processing chains unless deleting one
if chain_user_id.startswith(SYS_USER):
if chain_user_id.startswith(SYS_DELETER): # in system processing, TBD
for i in range(retries):
time.sleep(period)
if cluster_handler.get_by_id(chain_id).get("user_id") != \
chain_user_id:
return
logger.info("Delete in-deleting chain {}/{}".format(
chain_name, chain_id))
cluster_handler.delete(chain_id)
return
logger.info("will refresh health")
# free or used by user, then check its health
for i in range(retries):
if cluster_handler.refresh_health(chain_id): # chain is healthy
return
else:
time.sleep(period)
logger.warning("Chain {}/{} is unhealthy!".format(chain_name, chain_id))
# only reset free chains
if cluster_handler.get_by_id(chain_id).get("user_id") == "":
logger.info("Resetting free unhealthy chain {}/{}".format(
chain_name, chain_id))
cluster_handler.reset_free_one(chain_id)
def cluster_query(cluster_id):
"""Query a json obj of a cluster
GET /cluster/xxxx
Return a json obj of the cluster.
"""
request_debug(r, logger)
result = cluster_handler.get_by_id(cluster_id)
if result:
return make_ok_resp(data=result)
else:
error_msg = "cluster not found with id=" + cluster_id
logger.warning(error_msg)
return make_fail_resp(error=error_msg, data=r.form,
code=CODE_NOT_FOUND)
def cluster_info():
"""
Return information of the cluster.
"""
request_debug(r, logger)
cluster_id = request_get(r, "cluster_id")
logger.warning("cluster_id={}".format(cluster_id))
if not cluster_id:
response_fail["error"] = "cluster_get without cluster_id"
logger.warning(response_fail["error"])
response_fail["data"] = r.args
return jsonify(response_fail), CODE_BAD_REQUEST
result = cluster_handler.get_by_id(cluster_id)
response_ok["data"] = result
return jsonify(response_ok), CODE_OK
def cluster_api():
logger.info("/cluster action=" + r.method)
request_debug(r, logger)
if r.method == 'GET':
if not r.form["id"]:
logger.warn("cluster get without enough data")
response_fail["error"] = "cluster GET without enough data"
response_fail["data"] = r.form
return jsonify(response_fail), CODE_BAD_REQUEST
else:
logger.debug("id=" + r.form['id'])
result = cluster_handler.get_by_id(r.form['id'])
if result:
return jsonify(result), CODE_OK
else:
logger.warn("cluster not found with id=" + r.form['id'])
response_fail["data"] = r.form
return jsonify(response_fail), CODE_BAD_REQUEST
elif r.method == 'POST':
if not r.form["name"] or not r.form["host_id"] or not \
r.form["consensus_plugin"] or not r.form["size"]:
logger.warn("cluster post without enough data")
response_fail["error"] = "cluster POST without enough data"
response_fail["data"] = r.form
return jsonify(response_fail), CODE_BAD_REQUEST
else:
name, host_id, consensus_plugin, consensus_mode, size = \
r.form['name'], r.form['host_id'], r.form['consensus_plugin'],\
r.form['consensus_mode'] or CONSENSUS_MODES[0], int(r.form[
"size"])
if consensus_plugin not in CONSENSUS_PLUGINS:
logger.debug(
"Unknown consensus_plugin={}".format(consensus_plugin))
return jsonify(response_fail), CODE_BAD_REQUEST
if consensus_plugin != CONSENSUS_PLUGINS[0] and consensus_mode \
not in CONSENSUS_MODES:
logger.debug("Invalid consensus, plugin={}, mode={}".format(
consensus_plugin, consensus_mode))
return jsonify(response_fail), CODE_BAD_REQUEST
if size not in CLUSTER_SIZES:
logger.debug("Unknown cluster size={}".format(size))
return jsonify(response_fail), CODE_BAD_REQUEST
if cluster_handler.create(name=name,
host_id=host_id,
consensus_plugin=consensus_plugin,
consensus_mode=consensus_mode,
size=size):
logger.debug("cluster POST successfully")
return jsonify(response_ok), CODE_CREATED
else:
logger.debug("cluster creation failed")
response_fail["error"] = "Failed to create cluster {}".format(
name)
return jsonify(response_fail), CODE_BAD_REQUEST
elif r.method == 'DELETE':
if not r.form["id"] or not r.form["col_name"]:
logger.warn("cluster operation post without enough data")
response_fail["error"] = "cluster delete without enough data"
response_fail["data"] = r.form
return jsonify(response_fail), CODE_BAD_REQUEST
else:
logger.debug("cluster delete with id={0}, col_name={1}".format(
r.form["id"], r.form["col_name"]))
if r.form["col_name"] == "active":
result = cluster_handler.delete(id=r.form["id"])
else:
result = cluster_handler.delete_released(id=r.form["id"])
if result:
return jsonify(response_ok), CODE_OK
else:
logger.debug("cluster deletion failed")
response_fail["error"] = "Failed to delete cluster {}".format(
r.form["id"])
return jsonify(response_fail), CODE_BAD_REQUEST
else:
response_fail["error"] = "unknown operation method"
response_fail["data"] = r.form
return jsonify(response_fail), CODE_BAD_REQUEST
