def init(job_id=None,
mode: WorkMode = WorkMode.STANDALONE,
naming_policy: NamingPolicy = NamingPolicy.DEFAULT):
if RuntimeInstance.EGGROLL:
return
if job_id is None:
job_id = str(uuid.uuid1())
LoggerFactory.setDirectory()
else:
LoggerFactory.setDirectory(
os.path.join(file_utils.get_project_base_directory(), 'logs',
job_id))
RuntimeInstance.MODE = mode
eggroll_context = EggRollContext(naming_policy=naming_policy)
if mode == WorkMode.STANDALONE:
from eggroll.api.standalone.eggroll import Standalone
RuntimeInstance.EGGROLL = Standalone(job_id=job_id,
eggroll_context=eggroll_context)
elif mode == WorkMode.CLUSTER:
from eggroll.api.cluster.eggroll import _EggRoll
from eggroll.api.cluster.eggroll import init as c_init
c_init(job_id, eggroll_context=eggroll_context)
RuntimeInstance.EGGROLL = _EggRoll.get_instance()
else:
from eggroll.api.cluster import simple_roll
simple_roll.init(job_id)
RuntimeInstance.EGGROLL = simple_roll.EggRoll.get_instance()
RuntimeInstance.EGGROLL.table("__clustercomm__", job_id, partition=10)
def setDirectory(directory=None):
with LoggerFactory.lock:
if not directory:
directory = os.path.join(file_utils.get_project_base_directory(), 'logs')
LoggerFactory.LOG_DIR = directory
os.makedirs(LoggerFactory.LOG_DIR, exist_ok=True)
for className, (logger, handler) in LoggerFactory.loggerDict.items():
logger.removeHandler(handler)
handler.close()
_hanlder = LoggerFactory.get_hanlder(className)
logger.addHandler(_hanlder)
LoggerFactory.loggerDict[className] = logger, _hanlder
def destroy(self):
for p in range(self._partitions):
env = self._get_env_for_partition(p, write=True)
db = env.open_db()
with env.begin(write=True) as txn:
txn.drop(db)
_table_key = ".".join([self._type, self._namespace, self._name])
Standalone.get_instance().meta_table.delete(_table_key)
_path = _get_db_path(self._type, self._namespace, self._name)
file_lock = str(file_utils.get_project_base_directory() + "/data/" +
"_".join([self._type, self._namespace, self._name]) +
".lock")
if not os.path.exists(file_lock):
fd = os.open(file_lock, os.O_CREAT)
shutil.rmtree(_path, ignore_errors=True)
if os.path.exists(file_lock):
os.remove(file_lock)
def __init__(self, eggroll_session):
self.data_dir = os.path.join(file_utils.get_project_base_directory(),
'data')
self.session_id = eggroll_session.get_session_id()
self.meta_table = _DTable('__META__', '__META__', 'fragments', 10)
self.pool = Executor()
Standalone.__instance = self
self.eggroll_session = eggroll_session
self.unique_id_template = '_EggRoll_%s_%s_%s_%.20f_%d'
# todo: move to eggrollSession
try:
self.host_name = socket.gethostname()
self.host_ip = socket.gethostbyname(self.host_name)
except socket.gaierror as e:
self.host_name = 'unknown'
self.host_ip = 'unknown'
def __init__(self, job_id=None, eggroll_context=None):
self.data_dir = os.path.join(file_utils.get_project_base_directory(), 'data')
self.job_id = str(uuid.uuid1()) if job_id is None else "{}".format(job_id)
self.meta_table = _DTable('__META__', '__META__', 'fragments', 10)
self.pool = Executor()
Standalone.__instance = self
if not eggroll_context:
eggroll_context = EggRollContext()
self.eggroll_context = eggroll_context
self.unique_id_template = '_EggRoll_%s_%s_%s_%.20f_%d'
# todo: move to EggRollContext
try:
self.host_name = socket.gethostname()
self.host_ip = socket.gethostbyname(self.host_name)
except socket.gaierror as e:
self.host_name = 'unknown'
self.host_ip = 'unknown'
def init(session_id=None,
mode: WorkMode = WorkMode.STANDALONE,
server_conf_path="eggroll/conf/server_conf.json",
eggroll_session: EggrollSession = None,
computing_engine_conf=None,
naming_policy=NamingPolicy.DEFAULT,
tag=None,
job_id=None,
chunk_size=100000):
if RuntimeInstance.EGGROLL:
return
if not session_id:
session_id = str(uuid.uuid1())
LoggerFactory.setDirectory(
os.path.join(file_utils.get_project_base_directory(), 'logs',
session_id))
if not job_id:
job_id = session_id
RuntimeInstance.MODE = mode
eggroll_session = EggrollSession(session_id=session_id,
naming_policy=naming_policy)
if mode == WorkMode.STANDALONE:
from eggroll.api.standalone.eggroll import Standalone
RuntimeInstance.EGGROLL = Standalone(eggroll_session=eggroll_session)
elif mode == WorkMode.CLUSTER:
from eggroll.api.cluster.eggroll import _EggRoll
from eggroll.api.cluster.eggroll import init as c_init
c_init(session_id=session_id,
server_conf_path=server_conf_path,
computing_engine_conf=computing_engine_conf,
naming_policy=naming_policy,
tag=tag,
job_id=job_id)
RuntimeInstance.EGGROLL = _EggRoll.get_instance()
else:
from eggroll.api.cluster import simple_roll
simple_roll.init(job_id)
RuntimeInstance.EGGROLL = simple_roll.EggRoll.get_instance()
RuntimeInstance.EGGROLL.table("__clustercomm__", job_id, partition=10)
