def init(): if EggRoll.init_flag: return config = file_utils.load_json_conf('eggroll/conf/mock_roll.json') egg_ids = config.get('eggs') for egg_id in egg_ids: target = config.get('storage').get(egg_id) channel = grpc.insecure_channel( target, options=[('grpc.max_send_message_length', -1), ('grpc.max_receive_message_length', -1)]) EggRoll.egg_list.append(kv_pb2_grpc.KVServiceStub(channel)) procs = config.get('procs').get(egg_id) for proc in procs: _channel = grpc.insecure_channel( proc, options=[('grpc.max_send_message_length', -1), ('grpc.max_receive_message_length', -1)]) _stub = processor_pb2_grpc.ProcessServiceStub(_channel) proc_info = (_channel, _stub) i = len(EggRoll.proc_list) EggRoll.proc_egg_map[i] = int(egg_id) - 1 EggRoll.proc_list.append(proc_info) EggRoll.init_flag = True
def __init__(self, job_id, party_id, role, runtime_conf, trans_conf="eggroll/conf/transfer_conf.json"): self.trans_conf = file_utils.load_json_conf(trans_conf) self.job_id = job_id self.party_id = party_id self.role = role self.runtime_conf = runtime_conf self._loop = asyncio.get_event_loop() ClusterCommRuntime.instance = self
def __init__(self, job_id, party_id, role, runtime_conf, host, port): self.trans_conf = file_utils.load_json_conf('conf/transfer_conf.json') self.job_id = job_id self.party_id = party_id self.role = role self.runtime_conf = runtime_conf self.channel = grpc.insecure_channel( target="{}:{}".format(host, port), options=[('grpc.max_send_message_length', -1), ('grpc.max_receive_message_length', -1)]) self.stub = cluster_comm_pb2_grpc.TransferSubmitServiceStub(self.channel) self.__pool = concurrent.futures.ThreadPoolExecutor() ClusterCommRuntime.instance = self
def init(job_id=None, server_conf_path="arch/conf/server_conf.json", eggroll_context=None): if job_id is None: job_id = str(uuid.uuid1()) global LOGGER LOGGER = getLogger() server_conf = file_utils.load_json_conf(server_conf_path) _roll_host = server_conf.get("servers").get("roll").get("host") _roll_port = server_conf.get("servers").get("roll").get("port") if not eggroll_context: eggroll_context = EggRollContext() _EggRoll(job_id, _roll_host, _roll_port, eggroll_context=eggroll_context)
def init(job_id, runtime_conf, server_conf_path): global LOGGER LOGGER = getLogger() server_conf = file_utils.load_json_conf(server_conf_path) if CONF_KEY_SERVER not in server_conf: raise EnvironmentError("server_conf should contain key {}".format(CONF_KEY_SERVER)) if CONF_KEY_TARGET not in server_conf.get(CONF_KEY_SERVER): raise EnvironmentError( "The {} should be a json file containing key: {}".format(server_conf_path, CONF_KEY_TARGET)) _host = server_conf.get(CONF_KEY_SERVER).get(CONF_KEY_TARGET).get("host") _port = server_conf.get(CONF_KEY_SERVER).get(CONF_KEY_TARGET).get("port") if CONF_KEY_LOCAL not in runtime_conf: raise EnvironmentError("runtime_conf should be a dict containing key: {}".format(CONF_KEY_LOCAL)) _party_id = runtime_conf.get(CONF_KEY_LOCAL).get("party_id") _role = runtime_conf.get(CONF_KEY_LOCAL).get("role") return ClusterCommRuntime(job_id, _party_id, _role, runtime_conf, _host, _port)
def init(session_id=None, server_conf_path="eggroll/conf/server_conf.json", eggroll_session=None, computing_engine_conf=None, naming_policy=NamingPolicy.DEFAULT, tag=None, job_id=None, chunk_size=CHUNK_SIZE_DEFAULT): if session_id is None: if job_id is not None: session_id = job_id else: session_id = str(uuid.uuid1()) if job_id is None: job_id = session_id if chunk_size < CHUNK_SIZE_MIN: chunk_size = CHUNK_SIZE_DEFAULT global LOGGER LOGGER = getLogger() server_conf = file_utils.load_json_conf(server_conf_path) if not eggroll_session: eggroll_session = EggrollSession( session_id=session_id, chunk_size=chunk_size, computing_engine_conf=computing_engine_conf, naming_policy=naming_policy, tag=tag) eggroll_session.add_conf('eggroll.server.conf.path', server_conf_path) eggroll_session.add_conf( EGGROLL_ROLL_HOST, server_conf.get("servers").get("roll").get("host")) eggroll_session.add_conf( EGGROLL_ROLL_PORT, server_conf.get("servers").get("roll").get("port")) eggroll_runtime = _EggRoll(eggroll_session=eggroll_session) eggroll_session.set_runtime(ComputingEngine.EGGROLL_DTABLE, eggroll_runtime) eggroll_session.set_gc_table(eggroll_runtime) eggroll_session.add_cleanup_task(eggroll_session.clean_duplicated_table)
def session_init(session_id=None, server_conf_path="eggroll/conf/server_conf.json", eggroll_session=None, computing_engine_conf=None, naming_policy=NamingPolicy.DEFAULT, tag=None, job_id=None, chunk_size=CHUNK_SIZE_DEFAULT): if session_id is None: if job_id is not None: session_id = job_id else: session_id = str(uuid.uuid1()) if job_id is None: job_id = session_id if chunk_size < CHUNK_SIZE_MIN: chunk_size = CHUNK_SIZE_DEFAULT global LOGGER LOGGER = getLogger() server_conf = file_utils.load_json_conf(server_conf_path) if not eggroll_session: eggroll_session = EggrollSession( session_id=session_id, chunk_size=chunk_size, computing_engine_conf=computing_engine_conf, naming_policy=naming_policy, tag=tag) eggroll_session.add_conf('eggroll.server.conf.path', server_conf_path) eggroll_session.add_conf( EGGROLL_ROLL_HOST, server_conf.get("servers").get("roll").get("host")) eggroll_session.add_conf( EGGROLL_ROLL_PORT, server_conf.get("servers").get("roll").get("port")) return eggroll_session