def get_standalone_context():
options = {
SessionConfKeys.CONFKEY_SESSION_DEPLOY_MODE: DeployModes.STANDALONE
}
session = ErSession(options=options)
print(session.get_session_id())
return session
def get_standalone_context(options=None):
if options is None:
options = {}
options[SessionConfKeys.CONFKEY_SESSION_DEPLOY_MODE] = DeployModes.STANDALONE
session = ErSession(options=options)
print(session.get_session_id())
context = RollPairContext(session)
return context
def __init__(self, session: ErSession):
self.__session = session
self.session_id = session.get_session_id()
self.default_store_type = StoreTypes.ROLLPAIR_LMDB
self.default_store_serdes = SerdesTypes.PICKLE
self.deploy_mode = session.get_option(
SessionConfKeys.CONFKEY_SESSION_DEPLOY_MODE)
self.__session_meta = session.get_session_meta()
self.__session.add_exit_task(self.context_gc)
self.rpc_gc_enable = True
self.gc_recorder = GcRecorder(self)
def get_cluster_context(options=None):
if options is None:
options = {}
if 'session_id' in options:
session_id = options['session_id']
else:
session_id = None
session = ErSession(session_id=session_id, options=options)
print(session.get_session_id())
context = RollPairContext(session)
return context
def __init__(self, session: ErSession):
if session.get_session_meta()._status != SessionStatus.ACTIVE:
raise Exception(
f"session_id={session.get_session_id()} is not ACTIVE. current status={session.get_session_meta()._status}"
)
self.__session = session
self.session_id = session.get_session_id()
default_store_type_str = RollPairConfKeys.EGGROLL_ROLLPAIR_DEFAULT_STORE_TYPE.get_with(
session.get_all_options())
self.default_store_type = getattr(StoreTypes, default_store_type_str,
None)
if not self.default_store_type:
raise ValueError(
f'store type "{default_store_type_str}" not found for roll pair'
)
self.in_memory_output = RollPairConfKeys.EGGROLL_ROLLPAIR_IN_MEMORY_OUTPUT.get_with(
session.get_all_options())
if not self.default_store_type:
raise ValueError(
f'in_memory_output "{self.in_memory_output}" not found for roll pair'
)
self.default_store_serdes = SerdesTypes.PICKLE
self.__session_meta = session.get_session_meta()
self.__session.add_exit_task(self.context_gc)
self.rpc_gc_enable = True
self.gc_recorder = GcRecorder(self)
self.__command_client = CommandClient()
self.session_default_rp = self.load(name=self.session_id,
namespace=f'er_session_meta',
options={
'total_partitions':
session.get_eggs_count(),
'store_type':
StoreTypes.ROLLPAIR_CACHE,
'create_if_missing':
True
})
eggs = session.get_eggs()
def _broadcast_eggs(task: ErTask):
from eggroll.core.utils import add_runtime_storage
_input = task._inputs[0]
add_runtime_storage("__eggs", eggs)
L.debug(f"runtime_storage={get_runtime_storage('__eggs')}")
self.session_default_rp.with_stores(func=_broadcast_eggs)
def __init__(self, session: ErSession):
if session.get_session_meta()._status != SessionStatus.ACTIVE:
raise Exception(f"session_id={session.get_session_id()} is not ACTIVE. current status={session.get_session_meta()._status}")
self.__session = session
self.session_id = session.get_session_id()
default_store_type_str = RollPairConfKeys.EGGROLL_ROLLPAIR_DEFAULT_STORE_TYPE.get_with(session.get_all_options())
self.default_store_type = getattr(StoreTypes, default_store_type_str, None)
if not self.default_store_type:
raise ValueError(f'store type "{default_store_type_str}" not found for roll pair')
self.default_store_serdes = SerdesTypes.PICKLE
self.deploy_mode = session.get_option(SessionConfKeys.CONFKEY_SESSION_DEPLOY_MODE)
self.__session_meta = session.get_session_meta()
self.__session.add_exit_task(self.context_gc)
self.rpc_gc_enable = True
self.gc_recorder = GcRecorder(self)
self.__command_client = CommandClient()
def __init__(self, session: ErSession):
if session.get_session_meta()._status != SessionStatus.ACTIVE:
raise Exception(
f"session:{session.get_session_id()} is not available, init first!"
)
self.__session = session
self.session_id = session.get_session_id()
self.default_store_type = StoreTypes.ROLLPAIR_LMDB
self.default_store_serdes = SerdesTypes.PICKLE
self.deploy_mode = session.get_option(
SessionConfKeys.CONFKEY_SESSION_DEPLOY_MODE)
self.__session_meta = session.get_session_meta()
self.__session.add_exit_task(self.context_gc)
self.rpc_gc_enable = True
self.gc_recorder = GcRecorder(self)
self.__command_client = CommandClient()
def get_debug_test_context(is_standalone=False,
manager_port=4670,
egg_port=20001,
transfer_port=20002,
session_id='testing'):
manager_port = manager_port
egg_ports = [egg_port]
egg_transfer_ports = [transfer_port]
self_server_node_id = 2
options = {}
if is_standalone:
options[SessionConfKeys.CONFKEY_SESSION_DEPLOY_MODE] = "standalone"
options[TransferConfKeys.CONFKEY_TRANSFER_SERVICE_HOST] = "127.0.0.1"
options[TransferConfKeys.CONFKEY_TRANSFER_SERVICE_PORT] = str(
transfer_port)
options[ClusterManagerConfKeys.CONFKEY_CLUSTER_MANAGER_PORT] = str(
manager_port)
options[NodeManagerConfKeys.CONFKEY_NODE_MANAGER_PORT] = str(manager_port)
egg = ErProcessor(id=1,
server_node_id=self_server_node_id,
processor_type=ProcessorTypes.EGG_PAIR,
status=ProcessorStatus.RUNNING,
command_endpoint=ErEndpoint("127.0.0.1", egg_ports[0]),
transfer_endpoint=ErEndpoint("127.0.0.1",
egg_transfer_ports[0]))
roll = ErProcessor(id=1,
server_node_id=self_server_node_id,
processor_type=ProcessorTypes.ROLL_PAIR_MASTER,
status=ProcessorStatus.RUNNING,
command_endpoint=ErEndpoint("127.0.0.1", manager_port))
session = ErSession(session_id, processors=[egg, roll], options=options)
print(session.get_session_id())
return session
def get_cluster_context(options=None):
if options is None:
options = {}
session = ErSession(options=options)
print(session.get_session_id())
return session