from eggroll.core.command.command_model import ErCommandRequest
from eggroll.core.constants import StoreTypes
from eggroll.core.io.kv_adapter import RocksdbSortedKvAdapter
from eggroll.core.meta_model import ErStoreLocator, ErJob, ErStore, ErFunctor
from eggroll.core.proto import command_pb2_grpc
from eggroll.core.serdes import cloudpickle
from eggroll.roll_pair import create_serdes, create_adapter
from eggroll.roll_pair.roll_pair import RollPair
from eggroll.roll_pair.roll_pair import RollPairContext
from eggroll.utils import log_utils
import numpy as np
import os
from eggroll.utils.log_utils import get_logger
L = get_logger()
if os.environ.get("EGGROLL_RPT_ENGINE_MOCK", "0") == "1":
import eggroll.roll_paillier_tensor.rpt_py_engine as CPUEngine
import eggroll.roll_paillier_tensor.rpt_gpu_engine as GPUEngine
else:
import rptCPU as CPUEngine
import rptGPU as GPUEngine
class Ciper(object):
def genkey(self):
return CPUEngine.keygen()
class Tensor(object):
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
import json
from time import sleep
import psutil
from eggroll.utils.log_utils import get_logger
L = get_logger(filename='profile')
def get_system_metric(interval=1):
io_start = psutil.disk_io_counters()._asdict()
net_start = psutil.net_io_counters()._asdict()
cpu_percent = psutil.cpu_percent(interval=interval)
memory = psutil.virtual_memory()._asdict()
io_end = psutil.disk_io_counters()._asdict()
net_end = psutil.net_io_counters()._asdict()
result = {}
io = {}
for k, v in io_end.items():
io[k] = io_end[k] - io_start[k]
from eggroll.core.conf_keys import RollSiteConfKeys, \
CoreConfKeys
from eggroll.core.datastructure import create_executor_pool
from eggroll.core.grpc.factory import GrpcChannelFactory
from eggroll.core.meta_model import ErEndpoint
from eggroll.core.proto import proxy_pb2, proxy_pb2_grpc
from eggroll.core.serdes import eggroll_serdes
from eggroll.core.transfer_model import ErRollSiteHeader
from eggroll.roll_pair import create_adapter
from eggroll.roll_pair.roll_pair import RollPair, RollPairContext
from eggroll.roll_pair.task.storage import PutBatchTask, FINISH_STATUS
from eggroll.roll_pair.transfer_pair import TransferPair
from eggroll.utils import log_utils
from grpc import RpcError
L = log_utils.get_logger()
P = log_utils.get_logger('profile')
_serdes = eggroll_serdes.PickleSerdes
RS_KEY_DELIM = "#"
STATUS_TABLE_NAME = "__rs_status"
ERROR_STATES = [proxy_pb2.STOP, proxy_pb2.KILL]
RS_KEY_PREFIX = "__rsk"
CONF_KEY_TARGET = "rollsite"
CONF_KEY_LOCAL = "local"
CONF_KEY_SERVER = "servers"
class CountDownLatch(object):
def __init__(self, count):
self.count = count
from arch.api.transfer import Rubbish, Party, Federation
from eggroll.core.meta_model import ErEndpoint
from eggroll.roll_pair.roll_pair import RollPair
from eggroll.utils import file_utils
# from eggroll.utils import log_utils
# from arch.api.utils.log_utils import getLogger
#log_utils.setDirectory()
#LOGGER = log_utils.get_logger()
OBJECT_STORAGE_NAME = "__federation__"
STATUS_TABLE_NAME = "__status__"
from arch.api.utils import file_utils
from eggroll.utils import log_utils
LOGGER = log_utils.get_logger()
#LOGGER = getLogger()
def init_roll_site_context(runtime_conf, session_id):
from eggroll.roll_site.roll_site import RollSiteContext
from eggroll.roll_pair.roll_pair import RollPairContext
LOGGER.info("init_roll_site_context runtime_conf: {}".format(runtime_conf))
session_instance = FateSession.get_instance()._eggroll.get_session()
rp_context = RollPairContext(session_instance)
role = runtime_conf.get("local").get("role")
party_id = str(runtime_conf.get("local").get("party_id"))
_path = file_utils.get_project_base_directory(
) + "/arch/conf/server_conf.json"
