def init():
if EggRoll.init_flag:
return
config = file_utils.load_json_conf('eggroll/conf/mock_roll.json')
egg_ids = config.get('eggs')
for egg_id in egg_ids:
target = config.get('storage').get(egg_id)
channel = grpc.insecure_channel(
target,
options=[('grpc.max_send_message_length', -1),
('grpc.max_receive_message_length', -1)])
EggRoll.egg_list.append(kv_pb2_grpc.KVServiceStub(channel))
procs = config.get('procs').get(egg_id)
for proc in procs:
_channel = grpc.insecure_channel(
proc,
options=[('grpc.max_send_message_length', -1),
('grpc.max_receive_message_length', -1)])
_stub = processor_pb2_grpc.ProcessServiceStub(_channel)
proc_info = (_channel, _stub)
i = len(EggRoll.proc_list)
EggRoll.proc_egg_map[i] = int(egg_id) - 1
EggRoll.proc_list.append(proc_info)
EggRoll.init_flag = True
def __init__(self, job_id, party_id, role, runtime_conf, trans_conf="eggroll/conf/transfer_conf.json"):
self.trans_conf = file_utils.load_json_conf(trans_conf)
self.job_id = job_id
self.party_id = party_id
self.role = role
self.runtime_conf = runtime_conf
self._loop = asyncio.get_event_loop()
ClusterCommRuntime.instance = self
def __init__(self, job_id, party_id, role, runtime_conf, host, port):
self.trans_conf = file_utils.load_json_conf('conf/transfer_conf.json')
self.job_id = job_id
self.party_id = party_id
self.role = role
self.runtime_conf = runtime_conf
self.channel = grpc.insecure_channel(
target="{}:{}".format(host, port),
options=[('grpc.max_send_message_length', -1), ('grpc.max_receive_message_length', -1)])
self.stub = cluster_comm_pb2_grpc.TransferSubmitServiceStub(self.channel)
self.__pool = concurrent.futures.ThreadPoolExecutor()
ClusterCommRuntime.instance = self
def init(job_id=None, server_conf_path="arch/conf/server_conf.json", eggroll_context=None):
if job_id is None:
job_id = str(uuid.uuid1())
global LOGGER
LOGGER = getLogger()
server_conf = file_utils.load_json_conf(server_conf_path)
_roll_host = server_conf.get("servers").get("roll").get("host")
_roll_port = server_conf.get("servers").get("roll").get("port")
if not eggroll_context:
eggroll_context = EggRollContext()
_EggRoll(job_id, _roll_host, _roll_port, eggroll_context=eggroll_context)
def init(job_id, runtime_conf, server_conf_path):
global LOGGER
LOGGER = getLogger()
server_conf = file_utils.load_json_conf(server_conf_path)
if CONF_KEY_SERVER not in server_conf:
raise EnvironmentError("server_conf should contain key {}".format(CONF_KEY_SERVER))
if CONF_KEY_TARGET not in server_conf.get(CONF_KEY_SERVER):
raise EnvironmentError(
"The {} should be a json file containing key: {}".format(server_conf_path, CONF_KEY_TARGET))
_host = server_conf.get(CONF_KEY_SERVER).get(CONF_KEY_TARGET).get("host")
_port = server_conf.get(CONF_KEY_SERVER).get(CONF_KEY_TARGET).get("port")
if CONF_KEY_LOCAL not in runtime_conf:
raise EnvironmentError("runtime_conf should be a dict containing key: {}".format(CONF_KEY_LOCAL))
_party_id = runtime_conf.get(CONF_KEY_LOCAL).get("party_id")
_role = runtime_conf.get(CONF_KEY_LOCAL).get("role")
return ClusterCommRuntime(job_id, _party_id, _role, runtime_conf, _host, _port)
def init(session_id=None,
server_conf_path="eggroll/conf/server_conf.json",
eggroll_session=None,
computing_engine_conf=None,
naming_policy=NamingPolicy.DEFAULT,
tag=None,
job_id=None,
chunk_size=CHUNK_SIZE_DEFAULT):
if session_id is None:
if job_id is not None:
session_id = job_id
else:
session_id = str(uuid.uuid1())
if job_id is None:
job_id = session_id
if chunk_size < CHUNK_SIZE_MIN:
chunk_size = CHUNK_SIZE_DEFAULT
global LOGGER
LOGGER = getLogger()
server_conf = file_utils.load_json_conf(server_conf_path)
if not eggroll_session:
eggroll_session = EggrollSession(
session_id=session_id,
chunk_size=chunk_size,
computing_engine_conf=computing_engine_conf,
naming_policy=naming_policy,
tag=tag)
eggroll_session.add_conf('eggroll.server.conf.path', server_conf_path)
eggroll_session.add_conf(
EGGROLL_ROLL_HOST,
server_conf.get("servers").get("roll").get("host"))
eggroll_session.add_conf(
EGGROLL_ROLL_PORT,
server_conf.get("servers").get("roll").get("port"))
eggroll_runtime = _EggRoll(eggroll_session=eggroll_session)
eggroll_session.set_runtime(ComputingEngine.EGGROLL_DTABLE,
eggroll_runtime)
eggroll_session.set_gc_table(eggroll_runtime)
eggroll_session.add_cleanup_task(eggroll_session.clean_duplicated_table)
def session_init(session_id=None,
server_conf_path="eggroll/conf/server_conf.json",
eggroll_session=None,
computing_engine_conf=None,
naming_policy=NamingPolicy.DEFAULT,
tag=None,
job_id=None,
chunk_size=CHUNK_SIZE_DEFAULT):
if session_id is None:
if job_id is not None:
session_id = job_id
else:
session_id = str(uuid.uuid1())
if job_id is None:
job_id = session_id
if chunk_size < CHUNK_SIZE_MIN:
chunk_size = CHUNK_SIZE_DEFAULT
global LOGGER
LOGGER = getLogger()
server_conf = file_utils.load_json_conf(server_conf_path)
if not eggroll_session:
eggroll_session = EggrollSession(
session_id=session_id,
chunk_size=chunk_size,
computing_engine_conf=computing_engine_conf,
naming_policy=naming_policy,
tag=tag)
eggroll_session.add_conf('eggroll.server.conf.path', server_conf_path)
eggroll_session.add_conf(
EGGROLL_ROLL_HOST,
server_conf.get("servers").get("roll").get("host"))
eggroll_session.add_conf(
EGGROLL_ROLL_PORT,
server_conf.get("servers").get("roll").get("port"))
return eggroll_session