Ejemplo n.º 1
0
    def init(self, cluster="", job_name="", task_index=0, **kwargs):
        """ Initialize the graph with creating graph server instance with
    given cluster env info.

    Args:
      cluster (dict | josn str): Empty dict or string when Graph runs with
        local mode. Otherwise, cluster includes server_count, client_count
        and traker.
        server_count (int): count of servers.
        client_count (int): count of clients.
        traker (str): traker path.
      job_name (str): `client` or `server`, default empty means Graph runs
        with local mode.
      task_index (int): index of current server of client.
    """
        tracker = ""
        if not cluster:
            self._deploy_mode = 0
            pywrap.set_deploy_mode(self._deploy_mode)
            self._client = pywrap.in_memory_client()
            task_index = 0
            server_count = 1
        else:
            self._deploy_mode = 1
            pywrap.set_deploy_mode(self._deploy_mode)
            if isinstance(cluster, dict):
                cluster_spec = cluster
            elif isinstance(cluster, str):
                cluster_spec = json.loads(cluster)
            server_count = cluster_spec.get("server_count")
            client_count = cluster_spec.get("client_count")
            if not server_count or not client_count:
                raise ValueError("cluster is composed of server_count,"
                                 "worker_count and tracker")
            tracker = cluster_spec.get("tracker")

            pywrap.set_server_count(server_count)
            pywrap.set_client_count(client_count)
            if tracker:
                pywrap.set_tracker(tracker)

        if job_name == "client":
            pywrap.set_client_id(task_index)
            self._client = pywrap.rpc_client()
            self._server = None
        else:
            if job_name == "server":
                self._client = None
            if not tracker and kwargs.get("tracker"):
                tracker = kwargs["tracker"]
            if tracker:
                self._server = Server(task_index, server_count, tracker)
            else:
                self._server = Server(task_index, server_count)
            self._server.start()
            self._server.init(self._edge_sources, self._node_sources)
        return self
Ejemplo n.º 2
0
    def deploy_in_server_mode(self, task_index, cluster, job_name):
        if isinstance(cluster, dict):
            cluster_spec = cluster
        elif isinstance(cluster, str):
            cluster_spec = json.loads(cluster)
        else:
            raise ValueError("cluster must be dict or json string.")

        tracker = cluster_spec.get("tracker", "root://graphlearn")

        # parse servers
        server_count = cluster_spec.get("server_count")
        servers = cluster_spec.get("server")
        if servers:
            pywrap.set_server_hosts(servers)
            servers = servers.split(',')
            server_count = len(servers)

        # parse clients
        client_count = cluster_spec.get("client_count")
        clients = cluster_spec.get("client")
        if clients:
            client_count = len(clients.split(','))

        if not server_count or not client_count:
            raise ValueError("Invalid cluster schema")

        pywrap.set_server_count(server_count)
        pywrap.set_client_count(client_count)

        if job_name == "client":
            pywrap.set_tracker(tracker)
            pywrap.set_client_id(task_index)
            self._client = pywrap.rpc_client()
            self._server = None
        elif job_name == "server":
            self._client = None
            server_host = "0.0.0.0:0" if not servers else servers[task_index]
            self._server = Server(task_index, server_count, server_host,
                                  tracker)
            self._server.start()
            self._server.init(self._edge_sources, self._node_sources)
        else:
            raise ValueError(
                "Only support client and server job name in SERVER mode.")
Ejemplo n.º 3
0
    def deploy_in_worker_mode(self, tracker, hosts, task_index, task_count):
        if hosts:
            pywrap.set_server_hosts(hosts)
            hosts = hosts.split(',')
            host = hosts[task_index]
            task_count = len(hosts)
        else:
            host = "0.0.0.0:0"

        assert task_index < task_count

        pywrap.set_client_id(task_index)
        pywrap.set_client_count(task_count)
        pywrap.set_server_count(task_count)
        pywrap.set_tracker(tracker)

        self._client = pywrap.in_memory_client()
        self._server = Server(task_index, task_count, host, tracker)
        self._server.start()
        self._server.init(self._edge_sources, self._node_sources)
Ejemplo n.º 4
0
    def init(self,
             task_index=0,
             task_count=1,
             cluster="",
             job_name="",
             **kwargs):
        """ Initialize the graph with creating graph server instance with
    given cluster env info.

    Args:
      task_index (int): Current task index in in_memory mode or current
        server or client index in independent mode.
      task_count (int): Total task count in in_memory mode.
      cluster (dict | josn str): Empty dict or string when Graph runs with
        local mode. Otherwise, cluster includes server_count, client_count
        and tracker.
        server_count (int): count of servers.
        client_count (int): count of clients.
        tracker (str): tracker path.
      job_name (str): `client` or `server`, default empty means Graph runs
        with local mode.
    """
        # In memory mode, local or distribute.
        if not cluster:
            assert job_name == "", "Initialize local server with empty `cluster`."
            server_count = task_count
            tracker = kwargs.get("tracker", 'root://graphlearn')
            if server_count == 1:
                assert task_index == 0, "Local mode, task_index=0, task_count=1"
                self._deploy_mode = 0  # Local in_memory mode.
            else:
                assert isinstance(server_count, int)
                assert isinstance(task_index, int)
                assert server_count > task_index
                self._deploy_mode = 2  # Distribute in_memory mode.
                pywrap.set_server_count(server_count)
                pywrap.set_client_count(server_count)
                pywrap.set_tracker(tracker)
                pywrap.set_client_id(task_index)
            pywrap.set_deploy_mode(self._deploy_mode)
            self._client = pywrap.in_memory_client()
        else:  # Distribute independent mode.
            assert job_name in ("client", "server")
            if isinstance(cluster, dict):
                cluster_spec = cluster
            elif isinstance(cluster, str):
                cluster_spec = json.loads(cluster)
            else:
                raise ValueError("cluster must be dict or json string.")
            server_count = cluster_spec.get("server_count")
            client_count = cluster_spec.get("client_count")
            tracker = cluster_spec.get("tracker", 'root://graphlearn')
            if not server_count or not client_count:
                raise ValueError("cluster is composed of server_count,"
                                 "worker_count and tracker")
            self._deploy_mode = 1
            pywrap.set_deploy_mode(self._deploy_mode)
            pywrap.set_server_count(server_count)
            pywrap.set_client_count(client_count)

        os.system('mkdir -p {}'.format(tracker))

        if job_name == "client":
            pywrap.set_tracker(tracker)
            pywrap.set_client_id(task_index)
            self._client = pywrap.rpc_client()
            self._server = None
        else:
            if job_name == "server":
                self._client = None
            self._server = Server(task_index, server_count, tracker)
            self._server.start()
            self._server.init(self._edge_sources, self._node_sources)
        return self
Ejemplo n.º 5
0
    def init(self,
             task_index=0,
             task_count=1,
             cluster="",
             job_name="",
             **kwargs):
        """ Initialize the graph with creating graph server instance with
    given cluster env info.

    Args:
      task_index (int): Current task index in in_memory mode or current
        server or client index in independent mode.
      task_count (int): Total task count in in_memory mode.
      cluster (dict | josn str): Empty dict or string when Graph runs with
        local mode. Otherwise, cluster includes (server_count, client_count,
        tracker) or (server, client) or (server, client_count)
        server_count (int): count of servers.
        client_count (int): count of clients.
        tracker (str): tracker path.
        server (string): hosts of servers, split by ','.
      job_name (str): `client` or `server`, default empty means Graph runs
        with local mode.
      kwargs:
        tracker (string): tracker path for in-memory mode.
        hosts (string): hosts of servers for in-memory mode.
    """
        # In memory mode, local or distribute.
        if not job_name:
            assert cluster == ""
            tracker = kwargs.get("tracker", 'root://graphlearn')
            hosts = kwargs.get("hosts")
            host = "0.0.0.0:0"
            if hosts:
                pywrap.set_server_hosts(hosts)
                hosts = hosts.split(',')
                host = hosts[task_index]
                task_count = len(hosts)
            assert task_index < task_count

            # Local in-memory mode.
            if task_count == 1:
                pywrap.set_deploy_mode(0)
            # Distribute in-memory mode.
            else:
                pywrap.set_deploy_mode(2)
                pywrap.set_server_count(task_count)
                pywrap.set_client_count(task_count)
                pywrap.set_tracker(tracker)
                pywrap.set_client_id(task_index)

            self._client = pywrap.in_memory_client()
            self._server = Server(task_index, task_count, host, tracker)
            self._server.start()
            self._server.init(self._edge_sources, self._node_sources)

        # Distribute service mode.
        else:
            if isinstance(cluster, dict):
                cluster_spec = cluster
            elif isinstance(cluster, str):
                cluster_spec = json.loads(cluster)
            else:
                raise ValueError("cluster must be dict or json string.")

            tracker = cluster_spec.get("tracker", 'root://graphlearn')
            server_count = cluster_spec.get("server_count")
            servers = cluster_spec.get("server")
            if servers:
                pywrap.set_server_hosts(servers)
                servers = servers.split(',')
                server_count = len(servers)

            client_count = cluster_spec.get("client_count")
            clients = cluster_spec.get("client")
            if clients:
                client_count = len(clients.split(','))
            if not server_count or not client_count:
                raise ValueError(
                    "cluster is composed of"
                    " (server_count, client_count, tracker)"
                    " or (server, client) or (server, client_count)}")
            pywrap.set_server_count(server_count)
            pywrap.set_client_count(client_count)
            pywrap.set_deploy_mode(1)

            if job_name == "client":
                pywrap.set_tracker(tracker)
                pywrap.set_client_id(task_index)
                self._client = pywrap.rpc_client()
                self._server = None
            elif job_name == "server":
                self._client = None
                server_host = "0.0.0.0:0" if not servers else servers[
                    task_index]
                self._server = Server(task_index, server_count, server_host,
                                      tracker)
                self._server.start()
                self._server.init(self._edge_sources, self._node_sources)
            else:
                raise ValueError("Only support client and server for GL.")
        return self