def network_check_health(net_id, retries=60, period=5): """ Check the chain health. If not healthy, will reset the chain :param net_id: id of the chain :param retries: how many retries before thinking not health :param period: wait between two retries :return: """ net = cluster_handler.get_by_id(net_id) if not net: logger.warning("Not find chain {}".format(net_id)) return if net.get("status") != NETWORK_STATUS_RUNNING: # check running one return net_name = net.get("name") logger.debug("Chain {}/{}: checking health".format(net_name, net_id)) # free or used by user, then check its health for i in range(retries): if cluster_handler.refresh_health(net_id): # chain is healthy return else: logger.debug("Health Check {}: cluster {}/{} is unhealthy!".format( i, net_name, net_id)) time.sleep(period) logger.warning("Chain {}/{} is unhealthy!".format(net_name, net_id)) # only reset free chains if cluster_handler.get_by_id(net_id).get("user_id") == "": logger.info("Timeout....resetting free unhealthy chain {}/{}".format( net_name, net_id)) cluster_handler.reset_free_one(net_id)
def cluster_info_show(cluster_id): logger.debug("/ cluster_info/{}?released={} action={}".format( cluster_id, r.args.get('released', '0'), r.method)) released = (r.args.get('released', '0') != '0') if not released: return render_template("cluster_info.html", item=cluster_handler.get_by_id(cluster_id), consensus_plugins=CONSENSUS_PLUGINS) else: return render_template("cluster_info.html", item=cluster_handler.get_by_id( cluster_id, col_name="released"), consensus_plugins=CONSENSUS_PLUGINS)
def cluster_info_show(cluster_id): logger.debug("/ cluster_info/{}?released={} action={}".format( cluster_id, r.args.get('released', '0'), r.method)) released = (r.args.get('released', '0') != '0') if not released: return render_template("cluster_info.html", item=cluster_handler.get_by_id(cluster_id), consensus_plugins=CONSENSUS_PLUGINS_FABRIC_V1) else: return render_template("cluster_info.html", item=cluster_handler.get_by_id( cluster_id, col_name="released"), consensus_plugins=CONSENSUS_PLUGINS_FABRIC_V1)
def chain_check_health(chain_id, retries=3, period=5): """ Check the chain health. :param chain_id: id of the chain :param retries: how many retries before thinking not health :param period: wait between two retries :return: """ # if not cluster_handler.check_health(chain_id) \ # and c['user_id'] != SYS_UNHEALTHY: # cluster_handler.release_cluster(c['id'], record=False) chain = cluster_handler.get_by_id(chain_id) if not chain: logger.warning("Not find chain with id = {}".format(chain_id)) return chain_user_id = chain.get("user_id") chain_name = chain.get("name") logger.debug("Chain {}/{}: checking health".format(chain_name, chain_id)) # we should never process in-processing chains unless deleting one if chain_user_id.startswith(SYS_USER): if chain_user_id.startswith(SYS_DELETER): # in system processing, TBD for i in range(retries): time.sleep(period) if cluster_handler.get_by_id(chain_id).get("user_id") != \ chain_user_id: return logger.info("Delete in-deleting chain {}/{}".format( chain_name, chain_id)) cluster_handler.delete(chain_id) return # free or used by user, then check its health for i in range(retries): if cluster_handler.refresh_health(chain_id): # chain is healthy return else: time.sleep(period) logger.warning("Chain {}/{} is unhealthy!".format(chain_name, chain_id)) # only reset free chains if cluster_handler.get_by_id(chain_id).get("user_id") == "": logger.info("Deleting free unhealthy chain {}/{}".format( chain_name, chain_id)) # cluster_handler.delete(chain_id) cluster_handler.reset_free_one(chain_id)
def chain_check_health(chain_id, retries=3, period=5): """ Check the chain health. :param chain_id: id of the chain :param retries: how many retries before thinking not health :param period: wait between two retries :return: """ chain = cluster_handler.get_by_id(chain_id) if not chain: logger.warning("Not find chain {}".format(chain_id)) return chain_user_id = chain.get("user_id") chain_name = chain.get("name") logger.debug("Chain {}/{}: checking health".format(chain_name, chain_id)) # we should never process in-processing chains unless deleting one if chain_user_id.startswith(SYS_USER): if chain_user_id.startswith(SYS_DELETER): # in system processing, TBD for i in range(retries): time.sleep(period) if cluster_handler.get_by_id(chain_id).get("user_id") != \ chain_user_id: return logger.info("Delete in-deleting chain {}/{}".format( chain_name, chain_id)) cluster_handler.delete(chain_id) return logger.info("will refresh health") # free or used by user, then check its health for i in range(retries): if cluster_handler.refresh_health(chain_id): # chain is healthy return else: time.sleep(period) logger.warning("Chain {}/{} is unhealthy!".format(chain_name, chain_id)) # only reset free chains if cluster_handler.get_by_id(chain_id).get("user_id") == "": logger.info("Resetting free unhealthy chain {}/{}".format( chain_name, chain_id)) cluster_handler.reset_free_one(chain_id)
def cluster_query(cluster_id): """Query a json obj of a cluster GET /cluster/xxxx Return a json obj of the cluster. """ request_debug(r, logger) result = cluster_handler.get_by_id(cluster_id) if result: return make_ok_resp(data=result) else: error_msg = "cluster not found with id=" + cluster_id logger.warning(error_msg) return make_fail_resp(error=error_msg, data=r.form, code=CODE_NOT_FOUND)
def cluster_info(): """ Return information of the cluster. """ request_debug(r, logger) cluster_id = request_get(r, "cluster_id") logger.warning("cluster_id={}".format(cluster_id)) if not cluster_id: response_fail["error"] = "cluster_get without cluster_id" logger.warning(response_fail["error"]) response_fail["data"] = r.args return jsonify(response_fail), CODE_BAD_REQUEST result = cluster_handler.get_by_id(cluster_id) response_ok["data"] = result return jsonify(response_ok), CODE_OK
def cluster_api(): logger.info("/cluster action=" + r.method) request_debug(r, logger) if r.method == 'GET': if not r.form["id"]: logger.warn("cluster get without enough data") response_fail["error"] = "cluster GET without enough data" response_fail["data"] = r.form return jsonify(response_fail), CODE_BAD_REQUEST else: logger.debug("id=" + r.form['id']) result = cluster_handler.get_by_id(r.form['id']) if result: return jsonify(result), CODE_OK else: logger.warn("cluster not found with id=" + r.form['id']) response_fail["data"] = r.form return jsonify(response_fail), CODE_BAD_REQUEST elif r.method == 'POST': if not r.form["name"] or not r.form["host_id"] or not \ r.form["consensus_plugin"] or not r.form["size"]: logger.warn("cluster post without enough data") response_fail["error"] = "cluster POST without enough data" response_fail["data"] = r.form return jsonify(response_fail), CODE_BAD_REQUEST else: name, host_id, consensus_plugin, consensus_mode, size = \ r.form['name'], r.form['host_id'], r.form['consensus_plugin'],\ r.form['consensus_mode'] or CONSENSUS_MODES[0], int(r.form[ "size"]) if consensus_plugin not in CONSENSUS_PLUGINS: logger.debug( "Unknown consensus_plugin={}".format(consensus_plugin)) return jsonify(response_fail), CODE_BAD_REQUEST if consensus_plugin != CONSENSUS_PLUGINS[0] and consensus_mode \ not in CONSENSUS_MODES: logger.debug("Invalid consensus, plugin={}, mode={}".format( consensus_plugin, consensus_mode)) return jsonify(response_fail), CODE_BAD_REQUEST if size not in CLUSTER_SIZES: logger.debug("Unknown cluster size={}".format(size)) return jsonify(response_fail), CODE_BAD_REQUEST if cluster_handler.create(name=name, host_id=host_id, consensus_plugin=consensus_plugin, consensus_mode=consensus_mode, size=size): logger.debug("cluster POST successfully") return jsonify(response_ok), CODE_CREATED else: logger.debug("cluster creation failed") response_fail["error"] = "Failed to create cluster {}".format( name) return jsonify(response_fail), CODE_BAD_REQUEST elif r.method == 'DELETE': if not r.form["id"] or not r.form["col_name"]: logger.warn("cluster operation post without enough data") response_fail["error"] = "cluster delete without enough data" response_fail["data"] = r.form return jsonify(response_fail), CODE_BAD_REQUEST else: logger.debug("cluster delete with id={0}, col_name={1}".format( r.form["id"], r.form["col_name"])) if r.form["col_name"] == "active": result = cluster_handler.delete(id=r.form["id"]) else: result = cluster_handler.delete_released(id=r.form["id"]) if result: return jsonify(response_ok), CODE_OK else: logger.debug("cluster deletion failed") response_fail["error"] = "Failed to delete cluster {}".format( r.form["id"]) return jsonify(response_fail), CODE_BAD_REQUEST else: response_fail["error"] = "unknown operation method" response_fail["data"] = r.form return jsonify(response_fail), CODE_BAD_REQUEST