def init(job_id, runtime_conf): global LOGGER LOGGER = getLogger() if CONF_KEY_LOCAL not in runtime_conf: raise EnvironmentError("runtime_conf should be a dict containing key: {}".format(CONF_KEY_LOCAL)) _party_id = runtime_conf.get(CONF_KEY_LOCAL).get('party_id') _role = runtime_conf.get(CONF_KEY_LOCAL).get("role") return ClusterCommRuntime(job_id, _party_id, _role, runtime_conf)
def init(job_id=None, server_conf_path="arch/conf/server_conf.json", eggroll_context=None): if job_id is None: job_id = str(uuid.uuid1()) global LOGGER LOGGER = getLogger() server_conf = file_utils.load_json_conf(server_conf_path) _roll_host = server_conf.get("servers").get("roll").get("host") _roll_port = server_conf.get("servers").get("roll").get("port") if not eggroll_context: eggroll_context = EggRollContext() _EggRoll(job_id, _roll_host, _roll_port, eggroll_context=eggroll_context)
def init(job_id, runtime_conf, server_conf_path): global LOGGER LOGGER = getLogger() server_conf = file_utils.load_json_conf(server_conf_path) if CONF_KEY_SERVER not in server_conf: raise EnvironmentError("server_conf should contain key {}".format(CONF_KEY_SERVER)) if CONF_KEY_TARGET not in server_conf.get(CONF_KEY_SERVER): raise EnvironmentError( "The {} should be a json file containing key: {}".format(server_conf_path, CONF_KEY_TARGET)) _host = server_conf.get(CONF_KEY_SERVER).get(CONF_KEY_TARGET).get("host") _port = server_conf.get(CONF_KEY_SERVER).get(CONF_KEY_TARGET).get("port") if CONF_KEY_LOCAL not in runtime_conf: raise EnvironmentError("runtime_conf should be a dict containing key: {}".format(CONF_KEY_LOCAL)) _party_id = runtime_conf.get(CONF_KEY_LOCAL).get("party_id") _role = runtime_conf.get(CONF_KEY_LOCAL).get("role") return ClusterCommRuntime(job_id, _party_id, _role, runtime_conf, _host, _port)
def init(session_id=None, server_conf_path="eggroll/conf/server_conf.json", eggroll_session=None, computing_engine_conf=None, naming_policy=NamingPolicy.DEFAULT, tag=None, job_id=None, chunk_size=CHUNK_SIZE_DEFAULT): if session_id is None: if job_id is not None: session_id = job_id else: session_id = str(uuid.uuid1()) if job_id is None: job_id = session_id if chunk_size < CHUNK_SIZE_MIN: chunk_size = CHUNK_SIZE_DEFAULT global LOGGER LOGGER = getLogger() server_conf = file_utils.load_json_conf(server_conf_path) if not eggroll_session: eggroll_session = EggrollSession( session_id=session_id, chunk_size=chunk_size, computing_engine_conf=computing_engine_conf, naming_policy=naming_policy, tag=tag) eggroll_session.add_conf('eggroll.server.conf.path', server_conf_path) eggroll_session.add_conf( EGGROLL_ROLL_HOST, server_conf.get("servers").get("roll").get("host")) eggroll_session.add_conf( EGGROLL_ROLL_PORT, server_conf.get("servers").get("roll").get("port")) eggroll_runtime = _EggRoll(eggroll_session=eggroll_session) eggroll_session.set_runtime(ComputingEngine.EGGROLL_DTABLE, eggroll_runtime) eggroll_session.set_gc_table(eggroll_runtime) eggroll_session.add_cleanup_task(eggroll_session.clean_duplicated_table)
def session_init(session_id=None, server_conf_path="eggroll/conf/server_conf.json", eggroll_session=None, computing_engine_conf=None, naming_policy=NamingPolicy.DEFAULT, tag=None, job_id=None, chunk_size=CHUNK_SIZE_DEFAULT): if session_id is None: if job_id is not None: session_id = job_id else: session_id = str(uuid.uuid1()) if job_id is None: job_id = session_id if chunk_size < CHUNK_SIZE_MIN: chunk_size = CHUNK_SIZE_DEFAULT global LOGGER LOGGER = getLogger() server_conf = file_utils.load_json_conf(server_conf_path) if not eggroll_session: eggroll_session = EggrollSession( session_id=session_id, chunk_size=chunk_size, computing_engine_conf=computing_engine_conf, naming_policy=naming_policy, tag=tag) eggroll_session.add_conf('eggroll.server.conf.path', server_conf_path) eggroll_session.add_conf( EGGROLL_ROLL_HOST, server_conf.get("servers").get("roll").get("host")) eggroll_session.add_conf( EGGROLL_ROLL_PORT, server_conf.get("servers").get("roll").get("port")) return eggroll_session
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import grpc import threading from eggroll.api.proto import kv_pb2, kv_pb2_grpc from eggroll.api.utils import log_utils from eggroll.api.utils import eggroll_serdes log_utils.setDirectory() LOGGER = log_utils.getLogger() try: import rocksdb except: LOGGER.info("WRAN: failed to import rocksdb") try: import lmdb except: LOGGER.info("WRAN: failed to import lmdb") LMDB_MAP_SIZE = 16 * 4_096 * 244_140 # follows storage-service-cxx's config here DEFAULT_DB = b'main' DELIMETER = '-' DELIMETER_ENCODED = DELIMETER.encode()
# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import json import os import copy from eggroll.api.utils.log_utils import getLogger LOGGER = getLogger() class ParameterOverride(object): @staticmethod def override_parameter(default_runtime_dict, setting_conf, submit_dict, out_prefix): if default_runtime_dict is None or submit_dict is None: raise Exception("default runtime conf and submit conf should be a json file") _method = submit_dict['task'] _module = submit_dict['module'] _module_setting = setting_conf['module'].get(_module) if not _module_setting: