def build_eggroll_session(work_mode: WorkMode, job_id=None, server_conf_path="eggroll/conf/server_conf.json"):
if work_mode.is_standalone():
from arch.standalone.core import EggrollSession
import uuid
session_id = job_id or str(uuid.uuid1())
session = EggrollSession(session_id=session_id)
return session
elif work_mode.is_cluster():
from eggroll.api.cluster.eggroll import session_init
return session_init(session_id=job_id, server_conf_path=server_conf_path)
raise ValueError(f"work_mode: {work_mode} not supported!")
def build_eggroll_runtime(work_mode: WorkMode, eggroll_session):
if work_mode.is_standalone():
from arch.standalone.standalone.eggroll import Standalone
return Standalone(eggroll_session)
elif work_mode.is_cluster():
from eggroll.api.cluster.eggroll import eggroll_init, _EggRoll
if _EggRoll.instance is None:
return eggroll_init(eggroll_session)
else:
raise ValueError(f"work_mode: {work_mode} not supported!")
def init(job_id=None,
mode: typing.Union[int, WorkMode] = WorkMode.STANDALONE,
backend: typing.Union[int, Backend] = Backend.EGGROLL,
persistent_engine: StoreType = StoreType.LMDB,
set_log_dir=True):
if isinstance(mode, int):
mode = WorkMode(mode)
if isinstance(backend, int):
backend = Backend(backend)
if RuntimeInstance.SESSION:
return
if job_id is None:
job_id = str(uuid.uuid1())
if set_log_dir:
LoggerFactory.set_directory()
else:
if set_log_dir:
LoggerFactory.set_directory(
os.path.join(file_utils.get_project_base_directory(), 'logs',
job_id))
RuntimeInstance.MODE = mode
RuntimeInstance.Backend = backend
from arch.api.table.session import build_session
session = build_session(job_id=job_id, work_mode=mode, backend=backend)
RuntimeInstance.SESSION = session
def build_eggroll_runtime(work_mode: WorkMode, eggroll_session):
# if isinstance(eggroll_session, EggrollSession):
# if work_mode.is_standalone():
# from eggroll.api.standalone.eggroll import Standalone
# return Standalone(eggroll_session)
#
# elif work_mode.is_cluster():
# from eggroll.api.cluster.eggroll import eggroll_init, _EggRoll
# if _EggRoll.instance is None:
# return eggroll_init(eggroll_session)
# else:
# raise ValueError(f"work_mode: {work_mode} not supported!")
if isinstance(eggroll_session, ErSession):
if work_mode.is_standalone() or work_mode.is_cluster():
return RollPairContext(eggroll_session)
else:
raise ValueError(f"work_mode: {work_mode} not supported!")
else:
raise ValueError(f"eggroll_session: {type(eggroll_session)} not supported!")
def maybe_create_eggroll_client():
"""
a tricky way to set eggroll client which may be used by spark tasks.
WARM: This may be removed or adjusted in future!
"""
import pickle
from pyspark.taskcontext import TaskContext
mode, eggroll_session = pickle.loads(
bytes.fromhex(TaskContext.get().getLocalProperty(_EGGROLL_CLIENT)))
build_eggroll_runtime(WorkMode(mode), eggroll_session)
def init(job_id=None,
mode: typing.Union[int, WorkMode] = WorkMode.STANDALONE,
backend: typing.Union[int, Backend] = Backend.EGGROLL,
persistent_engine: str = StoreTypes.ROLLPAIR_LMDB,
eggroll_version=None,
set_log_dir=True,
options: dict = None):
"""
Initializes session, should be called before all.
Parameters
---------
job_id : string
job id and default table namespace of this runtime.
mode : WorkMode
set work mode,
- standalone: `WorkMode.STANDALONE` or 0
- cluster: `WorkMode.CLUSTER` or 1
backend : Backend
set computing backend,
- eggroll: `Backend.EGGROLL` or 0
- spark: `Backend.SAPRK` or 1
options : None or dict
additional options
Returns
-------
None
nothing returns
Examples
--------
>>> from arch.api import session, WorkMode, Backend
>>> session.init("a_job_id", WorkMode.Standalone, Backend.EGGROLL)
"""
if RuntimeInstance.SESSION:
return
if isinstance(mode, int):
mode = WorkMode(mode)
if isinstance(backend, int):
backend = Backend(backend)
if job_id is None:
job_id = str(uuid.uuid1())
if True:
LoggerFactory.set_directory()
else:
if set_log_dir:
LoggerFactory.set_directory(
os.path.join(file_utils.get_project_base_directory(), 'logs',
job_id))
if eggroll_version is None:
eggroll_version = _EGGROLL_VERSION
if backend.is_eggroll():
if eggroll_version < 2:
from arch.api.impl.based_1x import build
builder = build.Builder(session_id=job_id,
work_mode=mode,
persistent_engine=persistent_engine)
else:
from arch.api.impl.based_2x import build
builder = build.Builder(session_id=job_id,
work_mode=mode,
persistent_engine=persistent_engine,
options=options)
elif backend.is_spark():
if eggroll_version < 2:
from arch.api.impl.based_spark.based_1x import build
builder = build.Builder(session_id=job_id,
work_mode=mode,
persistent_engine=persistent_engine)
else:
from arch.api.impl.based_spark.based_2x import build
builder = build.Builder(session_id=job_id,
work_mode=mode,
persistent_engine=persistent_engine,
options=options)
else:
raise ValueError(f"backend: ${backend} unknown")
RuntimeInstance.MODE = mode
RuntimeInstance.BACKEND = backend
RuntimeInstance.BUILDER = builder
RuntimeInstance.SESSION = builder.build_session()