def init(session_id=None,
server_conf_path="eggroll/conf/server_conf.json",
eggroll_session=None,
computing_engine_conf=None,
naming_policy=NamingPolicy.DEFAULT,
tag=None,
job_id=None,
chunk_size=CHUNK_SIZE_DEFAULT):
if session_id is None:
if job_id is not None:
session_id = job_id
else:
session_id = str(uuid.uuid1())
if job_id is None:
job_id = session_id
if chunk_size < CHUNK_SIZE_MIN:
chunk_size = CHUNK_SIZE_DEFAULT
global LOGGER
LOGGER = getLogger()
server_conf = file_utils.load_json_conf(server_conf_path)
if not eggroll_session:
eggroll_session = EggrollSession(
session_id=session_id,
chunk_size=chunk_size,
computing_engine_conf=computing_engine_conf,
naming_policy=naming_policy,
tag=tag)
eggroll_session.add_conf('eggroll.server.conf.path', server_conf_path)
eggroll_session.add_conf(
EGGROLL_ROLL_HOST,
server_conf.get("servers").get("roll").get("host"))
eggroll_session.add_conf(
EGGROLL_ROLL_PORT,
server_conf.get("servers").get("roll").get("port"))
eggroll_runtime = _EggRoll(eggroll_session=eggroll_session)
eggroll_session.set_runtime(ComputingEngine.EGGROLL_DTABLE,
eggroll_runtime)
def build_eggroll_session(work_mode: WorkMode, job_id=None, server_conf_path="eggroll/conf/server_conf.json"):
if work_mode.is_standalone():
from eggroll.api.core import EggrollSession
import uuid
session_id = job_id or str(uuid.uuid1())
session = EggrollSession(session_id=session_id)
return session
elif work_mode.is_cluster():
from eggroll.api.cluster.eggroll import session_init
return session_init(session_id=job_id, server_conf_path=server_conf_path)
raise ValueError(f"work_mode: {work_mode} not supported!")
def init(session_id=None,
mode: WorkMode = WorkMode.STANDALONE,
server_conf_path="eggroll/conf/server_conf.json",
eggroll_session: EggrollSession = None,
computing_engine_conf=None,
naming_policy=NamingPolicy.DEFAULT,
tag=None,
job_id=None,
chunk_size=100000):
if RuntimeInstance.EGGROLL:
return
if not session_id:
session_id = str(uuid.uuid1())
LoggerFactory.setDirectory(
os.path.join(file_utils.get_project_base_directory(), 'logs',
session_id))
if not job_id:
job_id = session_id
RuntimeInstance.MODE = mode
eggroll_session = EggrollSession(session_id=session_id,
naming_policy=naming_policy)
if mode == WorkMode.STANDALONE:
from eggroll.api.standalone.eggroll import Standalone
RuntimeInstance.EGGROLL = Standalone(eggroll_session=eggroll_session)
elif mode == WorkMode.CLUSTER:
from eggroll.api.cluster.eggroll import _EggRoll
from eggroll.api.cluster.eggroll import init as c_init
c_init(session_id=session_id,
server_conf_path=server_conf_path,
computing_engine_conf=computing_engine_conf,
naming_policy=naming_policy,
tag=tag,
job_id=job_id)
RuntimeInstance.EGGROLL = _EggRoll.get_instance()
else:
from eggroll.api.cluster import simple_roll
simple_roll.init(job_id)
RuntimeInstance.EGGROLL = simple_roll.EggRoll.get_instance()
RuntimeInstance.EGGROLL.table("__clustercomm__", job_id, partition=10)