from eggroll.core.command.command_model import ErCommandRequest from eggroll.core.constants import StoreTypes from eggroll.core.io.kv_adapter import RocksdbSortedKvAdapter from eggroll.core.meta_model import ErStoreLocator, ErJob, ErStore, ErFunctor from eggroll.core.proto import command_pb2_grpc from eggroll.core.serdes import cloudpickle from eggroll.roll_pair import create_serdes, create_adapter from eggroll.roll_pair.roll_pair import RollPair from eggroll.roll_pair.roll_pair import RollPairContext from eggroll.utils import log_utils import numpy as np import os from eggroll.utils.log_utils import get_logger L = get_logger() if os.environ.get("EGGROLL_RPT_ENGINE_MOCK", "0") == "1": import eggroll.roll_paillier_tensor.rpt_py_engine as CPUEngine import eggroll.roll_paillier_tensor.rpt_gpu_engine as GPUEngine else: import rptCPU as CPUEngine import rptGPU as GPUEngine class Ciper(object): def genkey(self): return CPUEngine.keygen() class Tensor(object):
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # import json from time import sleep import psutil from eggroll.utils.log_utils import get_logger L = get_logger(filename='profile') def get_system_metric(interval=1): io_start = psutil.disk_io_counters()._asdict() net_start = psutil.net_io_counters()._asdict() cpu_percent = psutil.cpu_percent(interval=interval) memory = psutil.virtual_memory()._asdict() io_end = psutil.disk_io_counters()._asdict() net_end = psutil.net_io_counters()._asdict() result = {} io = {} for k, v in io_end.items(): io[k] = io_end[k] - io_start[k]
from eggroll.core.conf_keys import RollSiteConfKeys, \ CoreConfKeys from eggroll.core.datastructure import create_executor_pool from eggroll.core.grpc.factory import GrpcChannelFactory from eggroll.core.meta_model import ErEndpoint from eggroll.core.proto import proxy_pb2, proxy_pb2_grpc from eggroll.core.serdes import eggroll_serdes from eggroll.core.transfer_model import ErRollSiteHeader from eggroll.roll_pair import create_adapter from eggroll.roll_pair.roll_pair import RollPair, RollPairContext from eggroll.roll_pair.task.storage import PutBatchTask, FINISH_STATUS from eggroll.roll_pair.transfer_pair import TransferPair from eggroll.utils import log_utils from grpc import RpcError L = log_utils.get_logger() P = log_utils.get_logger('profile') _serdes = eggroll_serdes.PickleSerdes RS_KEY_DELIM = "#" STATUS_TABLE_NAME = "__rs_status" ERROR_STATES = [proxy_pb2.STOP, proxy_pb2.KILL] RS_KEY_PREFIX = "__rsk" CONF_KEY_TARGET = "rollsite" CONF_KEY_LOCAL = "local" CONF_KEY_SERVER = "servers" class CountDownLatch(object): def __init__(self, count): self.count = count
from arch.api.transfer import Rubbish, Party, Federation from eggroll.core.meta_model import ErEndpoint from eggroll.roll_pair.roll_pair import RollPair from eggroll.utils import file_utils # from eggroll.utils import log_utils # from arch.api.utils.log_utils import getLogger #log_utils.setDirectory() #LOGGER = log_utils.get_logger() OBJECT_STORAGE_NAME = "__federation__" STATUS_TABLE_NAME = "__status__" from arch.api.utils import file_utils from eggroll.utils import log_utils LOGGER = log_utils.get_logger() #LOGGER = getLogger() def init_roll_site_context(runtime_conf, session_id): from eggroll.roll_site.roll_site import RollSiteContext from eggroll.roll_pair.roll_pair import RollPairContext LOGGER.info("init_roll_site_context runtime_conf: {}".format(runtime_conf)) session_instance = FateSession.get_instance()._eggroll.get_session() rp_context = RollPairContext(session_instance) role = runtime_conf.get("local").get("role") party_id = str(runtime_conf.get("local").get("party_id")) _path = file_utils.get_project_base_directory( ) + "/arch/conf/server_conf.json"