コード例 #1
0
    def db_update_one(self, filter, operations, after=True, col="active"):
        """
        Update the data into the active db

        :param filter: Which instance to update, e.g., {"id": "xxx"}
        :param operations: data to update to db, e.g., {"$set": {}}
        :param after: return AFTER or BEFORE
        :param col: collection to operate on
        :return: The updated host json dict
        """
        state = CLUSTER_STATE.active.name if col == "active" \
            else CLUSTER_STATE.released.name
        filter.update({"state": state})
        logger.info("filter {} operations {}".format(filter, operations))
        kwargs = dict(('set__' + k, v) for (k, v) in operations.items())
        for k, v in kwargs.items():
            logger.info("k={}, v={}".format(k, v))
        try:
            ClusterModel.objects(id=filter.get("id")).update(upsert=True,
                                                             **kwargs)
            doc = ClusterModel.objects.get(id=filter.get("id"))
        except Exception as exc:
            logger.info("exception {}".format(exc.message))
            return None
        return doc
コード例 #2
0
ファイル: host.py プロジェクト: Randyshu2018/baas
    def clean(self, id):
        """
        Clean a host's free clusters.

        :param id: host id
        :return: True or False
        """
        logger.debug("clean host with id = {}".format(id))
        host = self.get_by_id(id)
        if not host:
            return False
        clusters = ClusterModel.objects(host=host)
        if host.status != "active":
            return False

        if len(clusters) <= 0:
            return True

        host = self.db_set_by_id(id, **{"autofill": False})
        schedulable_status = host.schedulable
        if schedulable_status:
            host = self.db_set_by_id(id, **{"schedulable": False})

        for cluster_item in clusters:
            cid = str(cluster_item.id)
            t = Thread(target=cluster.cluster_handler.delete, args=(cid,))
            t.start()
            time.sleep(0.2)

        if schedulable_status:
            self.db_set_by_id(id, **{"schedulable": schedulable_status})

        return True
コード例 #3
0
ファイル: host.py プロジェクト: Randyshu2018/baas
    def reset(self, id):
        """
        Clean a host's free clusters.

        :param id: host id
        :return: True or False
        """
        logger.debug("clean host with id = {}".format(id))
        host = self.get_by_id(id)
        if not host or ClusterModel.objects(host=host).count() < 0:
            logger.warning("No find resettable host with id ={}".format(id))
            return False
        host_type = host.type
        return self.host_agents[host_type].reset(host_type, host.worker_api)
コード例 #4
0
def host_actions():
    logger.info("/host_op, method=" + r.method)
    request_debug(r, logger)
    if r.content_type.startswith("application/json"):
        body = dict(r.get_json(force=True, silent=True))
    else:
        body = r.form

    host_id, action = body['id'], body['action']
    if not host_id or not action:
        error_msg = "host POST without enough data"
        logger.warning(error_msg)
        return make_fail_resp(error=error_msg,
                              data=body)
    else:
        if action == "fillup":
            if host_handler.fillup(host_id):
                logger.debug("fillup successfully")
                return make_ok_resp()
            else:
                error_msg = "Failed to fillup the host."
                logger.warning(error_msg)
                return make_fail_resp(error=error_msg, data=body)
        elif action == "clean":
            if host_handler.clean(host_id):
                logger.debug("clean successfully")
                return make_ok_resp()
            else:
                error_msg = "Failed to clean the host."
                logger.warning(error_msg)
                return make_fail_resp(error=error_msg, data=body)
        elif action == "reset":
            if host_handler.reset(host_id):
                logger.debug("reset successfully")
                try:
                    host_model = HostModel.objects.get(id=host_id)
                    clusters = ClusterModel.objects(host=host_model)
                    for cluster_item in clusters:
                        cluster_item.delete()
                except Exception:
                    pass
                return make_ok_resp()
            else:
                error_msg = "Failed to reset the host."
                logger.warning(error_msg)
                return make_fail_resp(error=error_msg, data=body)

    error_msg = "unknown host action={}".format(action)
    logger.warning(error_msg)
    return make_fail_resp(error=error_msg, data=body)
コード例 #5
0
    def list(self, filter_data={}, col_name="active"):
        """ List clusters with given criteria

        :param filter_data: Image with the filter properties
        :param col_name: Use data in which col_name
        :return: list of serialized doc
        """
        result = []
        if col_name in [e.name for e in CLUSTER_STATE]:
            logger.debug("List all {} clusters".format(col_name))
            filter_data.update({"state": col_name})
            clusters = ClusterModel.objects(__raw__=filter_data)
            result = self._schema(clusters, many=True)
        else:
            logger.warning("Unknown cluster col_name=" + col_name)
        return result
コード例 #6
0
    def fillup(self, id):
        """
        Fullfil a host with clusters to its capacity limit

        :param id: host id
        :return: True or False
        """
        logger.debug("Try fillup host {}".format(id))
        host = self.get_by_id(id)
        if not host:
            return False
        if host.status != "active":
            logger.warning("host {} is not active".format(id))
            return False
        clusters = ClusterModel.objects(host=host)
        num_new = host.capacity - len(clusters)
        if num_new <= 0:
            logger.warning("host {} already full".format(id))
            return True

        free_ports = cluster.cluster_handler.find_free_start_ports(id, num_new)
        logger.debug("Free_ports = {}".format(free_ports))

        def create_cluster_work(start_port):
            cluster_name = "{}_{}".format(
                host.name,
                int((start_port - CLUSTER_PORT_START) / CLUSTER_PORT_STEP))
            cluster_size = random.choice(NETWORK_SIZE_FABRIC_V1)
            config = FabricV1NetworkConfig(
                consensus_plugin=CONSENSUS_PLUGIN_SOLO, size=cluster_size)
            config.network_type = NETWORK_TYPE_FABRIC_V1_1
            cid = cluster.cluster_handler.create(name=cluster_name,
                                                 host_id=id,
                                                 config=config,
                                                 start_port=start_port)
            if cid:
                logger.debug("Create cluster {} with id={}".format(
                    cluster_name, cid))
            else:
                logger.warning("Create cluster failed")

        for p in free_ports:
            t = Thread(target=create_cluster_work, args=(p, ))
            t.start()
            time.sleep(0.2)

        return True
コード例 #7
0
    def _get_cluster_info(self, cid, config):
        cluster = ClusterModel.objects.get(id=cid)

        cluster_name = cluster.name
        kube_config = KubernetesOperation()._get_config_from_params(cluster
                                                                    .host
                                                                    .k8s_param)

        clusters_exists = ClusterModel.objects(host=cluster.host)
        ports_index = [service.port for service in ServicePort
                       .objects(cluster__in=clusters_exists)]

        nfsServer_ip = cluster.host.k8s_param.get('K8SNfsServer')
        consensus = config['consensus_plugin']

        return cluster, cluster_name, kube_config, ports_index, \
            nfsServer_ip, consensus
コード例 #8
0
    def find_free_start_ports(self, host_id, number):
        """ Find the first available port for a new cluster api

        This is NOT lock-free. Should keep simple, fast and safe!

        Check existing cluster records in the host, find available one.

        :param host_id: id of the host
        :param number: Number of ports to get
        :return: The port list, e.g., [7050, 7150, ...]
        """
        logger.debug("Find {} start ports for host {}".format(number, host_id))
        if number <= 0:
            logger.warning("number {} <= 0".format(number))
            return []
        host = self.host_handler.get_by_id(host_id)
        if not host:
            logger.warning("Cannot find host with id={}", host_id)
            return ""

        clusters_exists = ClusterModel.objects(host=host)
        # clusters_valid = list(filter(lambda c: c.get("service_url"),
        #                              clusters_exists))
        # ports_existed = list(map(
        #     lambda c: int(c["service_url"]["rest"].split(":")[-1]),
        #     clusters_valid))
        ports_existed = [
            service.port
            for service in ServicePort.objects(cluster__in=clusters_exists)
        ]

        logger.debug("The ports existed: {}".format(ports_existed))
        if len(ports_existed) + number >= 1000:
            logger.warning("Too much ports are already in used.")
            return []
        candidates = [
            CLUSTER_PORT_START + i * CLUSTER_PORT_STEP
            for i in range(len(ports_existed) + number)
        ]

        result = list(filter(lambda x: x not in ports_existed, candidates))

        logger.debug("Free ports are {}".format(result[:number]))
        return result[:number]
コード例 #9
0
ファイル: host.py プロジェクト: Randyshu2018/baas
    def delete(self, id):
        """ Delete a host instance

        :param id: id of the host to delete
        :return:
        """
        logger.debug("Delete a host with id={0}".format(id))

        try:
            h = HostModel.objects.get(id=id)
        except Exception:
            logger.warning("Cannot delete non-existed host")
            return False

        host_type = h.type

        if ClusterModel.objects(host=h).count():
            logger.warning("Host type not found.")
            return False

        elif (host_type == WORKER_TYPE_DOCKER or
              host_type == WORKER_TYPE_SWARM):
            self.host_agents[host_type].delete(h.worker_api)

        elif host_type == WORKER_TYPE_VSPHERE:
            if h.status == "pending":
                return False
            vmuuid = h.vcparam[utils.VMUUID]
            vcip = h.vcparam[utils.VCIP]
            vcusername = h.vcparam[utils.VCUSERNAME]
            vcpwd = h.vcparam[utils.VCPWD]
            vcport = h.vcparam[utils.VCPORT]
            self.host_agents[host_type].delete(vmuuid,
                                               vcip,
                                               vcusername,
                                               vcpwd,
                                               vcport)
        h.delete()
        return True
コード例 #10
0
ファイル: host.py プロジェクト: Randyshu2018/baas
    def update(self, id, d):
        """ Update a host's property

        TODO: may check when changing host type

        :param id: id of the host
        :param d: dict to use as updated values
        :return: serialized result or obj
        """
        logger.debug("Get a host with id=" + id)
        h_old = self.get_by_id(id)
        if not h_old:
            logger.warning("No host found with id=" + id)
            return {}

        if h_old.status == "pending":
            return {}

        if "worker_api" in d and not d["worker_api"].startswith("tcp://"):
            d["worker_api"] = "tcp://" + d["worker_api"]

        if "capacity" in d:
            d["capacity"] = int(d["capacity"])
        if d["capacity"] < ClusterModel.objects(host=h_old).count():
            logger.warning("Cannot set cap smaller than running clusters")
            return {}
        if "log_server" in d and "://" not in d["log_server"]:
            d["log_server"] = "udp://" + d["log_server"]
        if "log_type" in d and d["log_type"] == CLUSTER_LOG_TYPES[0]:
            d["log_server"] = ""
        if "autofill" in d:
            d["autofill"] = d["autofill"] == "on"
        if "schedulable" in d:
            d["schedulable"] = d["schedulable"] == "on"
        self.db_set_by_id(id, **d)
        h_new = self.get_by_id(id)
        return self._schema(h_new)
コード例 #11
0
    def create(self, name, host_id, config, start_port=0, user_id=""):
        """ Create a cluster based on given data

        TODO: maybe need other id generation mechanism
        Args:

            name: name of the cluster
            host_id: id of the host URL
            config: network configuration
            start_port: first service port for cluster, will generate
             if not given
            user_id: user_id of the cluster if start to be applied

        return: Id of the created cluster or None
        """
        logger.info("Create cluster {}, host_id={}, config={}, start_port={}, "
                    "user_id={}".format(name, host_id, config.get_data(),
                                        start_port, user_id))

        worker = self.host_handler.get_active_host_by_id(host_id)
        if not worker:
            logger.error("Cannot find available host to create new network")
            return None

        if ClusterModel.objects(host=worker).count() >= worker.capacity:
            logger.warning("host {} is already full".format(host_id))
            return None

        peer_num = int(config.get_data().get("size", 4))
        ca_num = 2 if peer_num > 1 else 1

        cid = uuid4().hex
        mapped_ports, peer_ports, ca_ports, orderer_ports, explorer_ports = \
            self.gen_ports_mapping(peer_num, ca_num, start_port, host_id)
        if not mapped_ports:
            logger.error("mapped_ports={}".format(mapped_ports))
            return None

        env_mapped_ports = dict(
            ((k + '_port').upper(), str(v)) for (k, v) in mapped_ports.items())

        network_type = config['network_type']
        net = {  # net is a blockchain network instance
            'id': cid,
            'name': name,
            'user_id': user_id,
            'worker_api': worker.worker_api,
            'network_type': network_type,  # e.g., fabric-1.0
            'env': env_mapped_ports,
            'status': NETWORK_STATUS_CREATING,
            'mapped_ports': mapped_ports,
            'service_url': {},  # e.g., {rest: xxx:7050, grpc: xxx:7051}
        }
        net.update(config.get_data())

        # try to start one cluster at the host
        cluster = ClusterModel(**net)
        cluster.host = worker
        cluster.save()
        # start cluster creation asynchronously for better user experience.
        t = Thread(target=self._create_cluster,
                   args=(cluster, cid, mapped_ports, worker, config, user_id,
                         peer_ports, ca_ports, orderer_ports, explorer_ports))
        t.start()
        return cid