def init(job_id, runtime_conf):
global LOGGER
LOGGER = getLogger()
if CONF_KEY_LOCAL not in runtime_conf:
raise EnvironmentError("runtime_conf should be a dict containing key: {}".format(CONF_KEY_LOCAL))
_party_id = runtime_conf.get(CONF_KEY_LOCAL).get('party_id')
_role = runtime_conf.get(CONF_KEY_LOCAL).get("role")
return ClusterCommRuntime(job_id, _party_id, _role, runtime_conf)
def init(job_id=None, server_conf_path="arch/conf/server_conf.json", eggroll_context=None):
if job_id is None:
job_id = str(uuid.uuid1())
global LOGGER
LOGGER = getLogger()
server_conf = file_utils.load_json_conf(server_conf_path)
_roll_host = server_conf.get("servers").get("roll").get("host")
_roll_port = server_conf.get("servers").get("roll").get("port")
if not eggroll_context:
eggroll_context = EggRollContext()
_EggRoll(job_id, _roll_host, _roll_port, eggroll_context=eggroll_context)
def init(job_id, runtime_conf, server_conf_path):
global LOGGER
LOGGER = getLogger()
server_conf = file_utils.load_json_conf(server_conf_path)
if CONF_KEY_SERVER not in server_conf:
raise EnvironmentError("server_conf should contain key {}".format(CONF_KEY_SERVER))
if CONF_KEY_TARGET not in server_conf.get(CONF_KEY_SERVER):
raise EnvironmentError(
"The {} should be a json file containing key: {}".format(server_conf_path, CONF_KEY_TARGET))
_host = server_conf.get(CONF_KEY_SERVER).get(CONF_KEY_TARGET).get("host")
_port = server_conf.get(CONF_KEY_SERVER).get(CONF_KEY_TARGET).get("port")
if CONF_KEY_LOCAL not in runtime_conf:
raise EnvironmentError("runtime_conf should be a dict containing key: {}".format(CONF_KEY_LOCAL))
_party_id = runtime_conf.get(CONF_KEY_LOCAL).get("party_id")
_role = runtime_conf.get(CONF_KEY_LOCAL).get("role")
return ClusterCommRuntime(job_id, _party_id, _role, runtime_conf, _host, _port)
def init(session_id=None,
server_conf_path="eggroll/conf/server_conf.json",
eggroll_session=None,
computing_engine_conf=None,
naming_policy=NamingPolicy.DEFAULT,
tag=None,
job_id=None,
chunk_size=CHUNK_SIZE_DEFAULT):
if session_id is None:
if job_id is not None:
session_id = job_id
else:
session_id = str(uuid.uuid1())
if job_id is None:
job_id = session_id
if chunk_size < CHUNK_SIZE_MIN:
chunk_size = CHUNK_SIZE_DEFAULT
global LOGGER
LOGGER = getLogger()
server_conf = file_utils.load_json_conf(server_conf_path)
if not eggroll_session:
eggroll_session = EggrollSession(
session_id=session_id,
chunk_size=chunk_size,
computing_engine_conf=computing_engine_conf,
naming_policy=naming_policy,
tag=tag)
eggroll_session.add_conf('eggroll.server.conf.path', server_conf_path)
eggroll_session.add_conf(
EGGROLL_ROLL_HOST,
server_conf.get("servers").get("roll").get("host"))
eggroll_session.add_conf(
EGGROLL_ROLL_PORT,
server_conf.get("servers").get("roll").get("port"))
eggroll_runtime = _EggRoll(eggroll_session=eggroll_session)
eggroll_session.set_runtime(ComputingEngine.EGGROLL_DTABLE,
eggroll_runtime)
eggroll_session.set_gc_table(eggroll_runtime)
eggroll_session.add_cleanup_task(eggroll_session.clean_duplicated_table)
def session_init(session_id=None,
server_conf_path="eggroll/conf/server_conf.json",
eggroll_session=None,
computing_engine_conf=None,
naming_policy=NamingPolicy.DEFAULT,
tag=None,
job_id=None,
chunk_size=CHUNK_SIZE_DEFAULT):
if session_id is None:
if job_id is not None:
session_id = job_id
else:
session_id = str(uuid.uuid1())
if job_id is None:
job_id = session_id
if chunk_size < CHUNK_SIZE_MIN:
chunk_size = CHUNK_SIZE_DEFAULT
global LOGGER
LOGGER = getLogger()
server_conf = file_utils.load_json_conf(server_conf_path)
if not eggroll_session:
eggroll_session = EggrollSession(
session_id=session_id,
chunk_size=chunk_size,
computing_engine_conf=computing_engine_conf,
naming_policy=naming_policy,
tag=tag)
eggroll_session.add_conf('eggroll.server.conf.path', server_conf_path)
eggroll_session.add_conf(
EGGROLL_ROLL_HOST,
server_conf.get("servers").get("roll").get("host"))
eggroll_session.add_conf(
EGGROLL_ROLL_PORT,
server_conf.get("servers").get("roll").get("port"))
return eggroll_session
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import grpc
import threading
from eggroll.api.proto import kv_pb2, kv_pb2_grpc
from eggroll.api.utils import log_utils
from eggroll.api.utils import eggroll_serdes
log_utils.setDirectory()
LOGGER = log_utils.getLogger()
try:
import rocksdb
except:
LOGGER.info("WRAN: failed to import rocksdb")
try:
import lmdb
except:
LOGGER.info("WRAN: failed to import lmdb")
LMDB_MAP_SIZE = 16 * 4_096 * 244_140 # follows storage-service-cxx's config here
DEFAULT_DB = b'main'
DELIMETER = '-'
DELIMETER_ENCODED = DELIMETER.encode()
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import json
import os
import copy
from eggroll.api.utils.log_utils import getLogger
LOGGER = getLogger()
class ParameterOverride(object):
@staticmethod
def override_parameter(default_runtime_dict, setting_conf, submit_dict, out_prefix):
if default_runtime_dict is None or submit_dict is None:
raise Exception("default runtime conf and submit conf should be a json file")
_method = submit_dict['task']
_module = submit_dict['module']
_module_setting = setting_conf['module'].get(_module)
if not _module_setting:
