def __init__(self,
name: str,
tag: str,
rs_ctx: RollSiteContext,
options: dict = None):
if options is None:
options = {}
self.name = name
self.tag = tag
self.ctx = rs_ctx
self.options = options
self.party_id = self.ctx.party_id
self.dst_host = self.ctx.proxy_endpoint._host
self.dst_port = self.ctx.proxy_endpoint._port
self.roll_site_session_id = self.ctx.roll_site_session_id
self.local_role = self.ctx.role
if RollSiteBase._receive_executor_pool is None:
receive_executor_pool_size = int(
RollSiteConfKeys.
EGGROLL_ROLLSITE_RECEIVE_EXECUTOR_POOL_MAX_SIZE.get_with(
options))
receive_executor_pool_type = CoreConfKeys.EGGROLL_CORE_DEFAULT_EXECUTOR_POOL.get_with(
options)
self._receive_executor_pool = create_executor_pool(
canonical_name=receive_executor_pool_type,
max_workers=receive_executor_pool_size,
thread_name_prefix="rollsite-client")
self._push_start_time = None
self._pull_start_time = None
self._is_standalone = self.ctx.is_standalone
L.debug(
f'inited RollSite. my party_id={self.ctx.party_id}. proxy endpoint={self.dst_host}:{self.dst_port}'
)
def start(self, options: dict = None):
if dict is None:
options = {}
_executor_pool_type = CoreConfKeys.EGGROLL_CORE_DEFAULT_EXECUTOR_POOL.get_with(
options)
server = grpc.server(
create_executor_pool(
canonical_name=_executor_pool_type,
max_workers=1,
thread_name_prefix="roll_pair_transfer_service"),
options=[(cygrpc.ChannelArgKey.max_send_message_length, -1),
(cygrpc.ChannelArgKey.max_receive_message_length, -1)])
transfer_servicer = GrpcTransferServicer()
transfer_pb2_grpc.add_TransferServiceServicer_to_server(
transfer_servicer, server)
port = options.get(TransferConfKeys.CONFKEY_TRANSFER_SERVICE_PORT, 0)
port = server.add_insecure_port(f'[::]:{port}')
L.info(f'transfer service started at port={port}')
print(f'transfer service started at port={port}')
server.start()
import time
time.sleep(1000000)
def __init__(self, transfer_id: str):
# params from __init__ params
self.__transfer_id = transfer_id
if TransferPair._executor_pool is None:
with TransferPair._executor_pool_lock:
if TransferPair._executor_pool is None:
_max_workers = int(RollPairConfKeys.EGGROLL_ROLLPAIR_TRANSFERPAIR_EXECUTOR_POOL_MAX_SIZE.get())
_thread_pool_type = CoreConfKeys.EGGROLL_CORE_DEFAULT_EXECUTOR_POOL.get()
TransferPair._executor_pool = create_executor_pool(
canonical_name=_thread_pool_type,
max_workers=_max_workers,
thread_name_prefix="transferpair_pool")
L.info(f'transfer pair _executor_pool max_workers={_max_workers}')
def __init__(self):
self._channel_factory = GrpcChannelFactory()
if CommandClient._executor_pool is None:
with CommandClient._executor_pool_lock:
if CommandClient._executor_pool is None:
_executor_pool_type = CoreConfKeys.EGGROLL_CORE_DEFAULT_EXECUTOR_POOL.get(
)
_max_workers = int(
CoreConfKeys.
EGGROLL_CORE_CLIENT_COMMAND_EXECUTOR_POOL_MAX_SIZE.get(
))
CommandClient._executor_pool = create_executor_pool(
canonical_name=_executor_pool_type,
max_workers=_max_workers,
thread_name_prefix="command_client")
def test_put_all_multi_thread(self):
executor_pool_type = CoreConfKeys.EGGROLL_CORE_DEFAULT_EXECUTOR_POOL.get(
)
exe = create_executor_pool(canonical_name=executor_pool_type,
max_workers=2)
exe.submit(self.test_put_all)
def serve(args):
prefix = 'v1/egg-pair'
set_data_dir(args.data_dir)
CommandRouter.get_instance().register(
service_name=f"{prefix}/runTask",
route_to_module_name="eggroll.roll_pair.egg_pair",
route_to_class_name="EggPair",
route_to_method_name="run_task")
max_workers = int(
RollPairConfKeys.
EGGROLL_ROLLPAIR_EGGPAIR_SERVER_EXECUTOR_POOL_MAX_SIZE.get())
executor_pool_type = CoreConfKeys.EGGROLL_CORE_DEFAULT_EXECUTOR_POOL.get()
command_server = grpc.server(
create_executor_pool(canonical_name=executor_pool_type,
max_workers=max_workers,
thread_name_prefix="eggpair-command-server"),
options=
[("grpc.max_metadata_size",
int(CoreConfKeys.
EGGROLL_CORE_GRPC_SERVER_CHANNEL_MAX_INBOUND_METADATA_SIZE.get())
),
('grpc.max_send_message_length',
int(CoreConfKeys.
EGGROLL_CORE_GRPC_SERVER_CHANNEL_MAX_INBOUND_MESSAGE_SIZE.get())
),
('grpc.max_receive_message_length',
int(CoreConfKeys.
EGGROLL_CORE_GRPC_SERVER_CHANNEL_MAX_INBOUND_MESSAGE_SIZE.get())
),
('grpc.keepalive_time_ms',
int(CoreConfKeys.CONFKEY_CORE_GRPC_CHANNEL_KEEPALIVE_TIME_SEC.get())
* 1000),
('grpc.keepalive_timeout_ms',
int(CoreConfKeys.
CONFKEY_CORE_GRPC_SERVER_CHANNEL_KEEPALIVE_TIMEOUT_SEC.get()) *
1000),
('grpc.keepalive_permit_without_calls',
int(CoreConfKeys.
CONFKEY_CORE_GRPC_SERVER_CHANNEL_KEEPALIVE_WITHOUT_CALLS_ENABLED.
get())),
('grpc.per_rpc_retry_buffer_size',
int(CoreConfKeys.CONFKEY_CORE_GRPC_SERVER_CHANNEL_RETRY_BUFFER_SIZE.
get())), ('grpc.so_reuseport', False)])
command_servicer = CommandServicer()
command_pb2_grpc.add_CommandServiceServicer_to_server(
command_servicer, command_server)
transfer_servicer = GrpcTransferServicer()
port = args.port
transfer_port = args.transfer_port
port = command_server.add_insecure_port(f'[::]:{port}')
if transfer_port == "-1":
transfer_server = command_server
transfer_port = port
transfer_pb2_grpc.add_TransferServiceServicer_to_server(
transfer_servicer, transfer_server)
else:
transfer_server_max_workers = int(
RollPairConfKeys.
EGGROLL_ROLLPAIR_EGGPAIR_DATA_SERVER_EXECUTOR_POOL_MAX_SIZE.get())
transfer_server = grpc.server(
create_executor_pool(canonical_name=executor_pool_type,
max_workers=transfer_server_max_workers,
thread_name_prefix="transfer_server"),
options=
[('grpc.max_metadata_size',
int(CoreConfKeys.
EGGROLL_CORE_GRPC_SERVER_CHANNEL_MAX_INBOUND_METADATA_SIZE.
get())),
('grpc.max_send_message_length',
int(CoreConfKeys.
EGGROLL_CORE_GRPC_SERVER_CHANNEL_MAX_INBOUND_MESSAGE_SIZE.
get())),
('grpc.max_receive_message_length',
int(CoreConfKeys.
EGGROLL_CORE_GRPC_SERVER_CHANNEL_MAX_INBOUND_MESSAGE_SIZE.
get())),
('grpc.keepalive_time_ms',
int(CoreConfKeys.
CONFKEY_CORE_GRPC_SERVER_CHANNEL_KEEPALIVE_WITHOUT_CALLS_ENABLED
.get()) * 1000),
('grpc.keepalive_timeout_ms',
int(CoreConfKeys.
CONFKEY_CORE_GRPC_SERVER_CHANNEL_KEEPALIVE_TIMEOUT_SEC.get())
* 1000),
('grpc.keepalive_permit_without_calls',
int(CoreConfKeys.
CONFKEY_CORE_GRPC_SERVER_CHANNEL_KEEPALIVE_WITHOUT_CALLS_ENABLED
.get())),
('grpc.per_rpc_retry_buffer_size',
int(CoreConfKeys.
CONFKEY_CORE_GRPC_SERVER_CHANNEL_RETRY_BUFFER_SIZE.get())),
('grpc.so_reuseport', False)])
transfer_port = transfer_server.add_insecure_port(
f'[::]:{transfer_port}')
transfer_pb2_grpc.add_TransferServiceServicer_to_server(
transfer_servicer, transfer_server)
transfer_server.start()
pid = os.getpid()
L.info(
f"starting egg_pair service, port: {port}, transfer port: {transfer_port}, pid: {pid}"
)
command_server.start()
cluster_manager = args.cluster_manager
myself = None
cluster_manager_client = None
if cluster_manager:
session_id = args.session_id
server_node_id = int(args.server_node_id)
static_er_conf = get_static_er_conf()
static_er_conf['server_node_id'] = server_node_id
if not session_id:
raise ValueError('session id is missing')
options = {SessionConfKeys.CONFKEY_SESSION_ID: args.session_id}
myself = ErProcessor(id=int(args.processor_id),
server_node_id=server_node_id,
processor_type=ProcessorTypes.EGG_PAIR,
command_endpoint=ErEndpoint(host='localhost',
port=port),
transfer_endpoint=ErEndpoint(host='localhost',
port=transfer_port),
pid=pid,
options=options,
status=ProcessorStatus.RUNNING)
cluster_manager_host, cluster_manager_port = cluster_manager.strip(
).split(':')
L.info(f'egg_pair cluster_manager: {cluster_manager}')
cluster_manager_client = ClusterManagerClient(
options={
ClusterManagerConfKeys.CONFKEY_CLUSTER_MANAGER_HOST:
cluster_manager_host,
ClusterManagerConfKeys.CONFKEY_CLUSTER_MANAGER_PORT:
cluster_manager_port
})
cluster_manager_client.heartbeat(myself)
if platform.system() == "Windows":
t1 = threading.Thread(target=stop_processor,
args=[cluster_manager_client, myself])
t1.start()
L.info(f'egg_pair started at port={port}, transfer_port={transfer_port}')
run = True
def exit_gracefully(signum, frame):
nonlocal run
run = False
L.info(
f'egg_pair {args.processor_id} at port={port}, transfer_port={transfer_port}, pid={pid} receives signum={signal.getsignal(signum)}, stopping gracefully.'
)
signal.signal(signal.SIGTERM, exit_gracefully)
signal.signal(signal.SIGINT, exit_gracefully)
while run:
time.sleep(1)
L.info(f'sending exit heartbeat to cm')
if cluster_manager:
myself._status = ProcessorStatus.STOPPED
cluster_manager_client.heartbeat(myself)
GrpcChannelFactory.shutdown_all_now()
L.info(f'closing RocksDB open dbs')
#todo:1: move to RocksdbAdapter and provide a cleanup method
from eggroll.core.pair_store.rocksdb import RocksdbAdapter
for path, db in RocksdbAdapter.db_dict.items():
del db
gc.collect()
L.info(f'system metric at exit: {get_system_metric(1)}')
L.info(
f'egg_pair {args.processor_id} at port={port}, transfer_port={transfer_port}, pid={pid} stopped gracefully'
)
def setUp(self) -> None:
self.__executor_pool = create_executor_pool(max_workers=5)
