def get_debug_test_context(is_standalone=False, manager_port=4670, egg_port=20001, transfer_port=20002, session_id='testing'): manager_port = manager_port egg_ports = [egg_port] egg_transfer_ports = [transfer_port] self_server_node_id = 2 options = {} if is_standalone: options[SessionConfKeys.CONFKEY_SESSION_DEPLOY_MODE] = "standalone" options[TransferConfKeys.CONFKEY_TRANSFER_SERVICE_HOST] = "127.0.0.1" options[TransferConfKeys.CONFKEY_TRANSFER_SERVICE_PORT] = str(transfer_port) options[ClusterManagerConfKeys.CONFKEY_CLUSTER_MANAGER_PORT] = str(manager_port) options[NodeManagerConfKeys.CONFKEY_NODE_MANAGER_PORT] = str(manager_port) egg = ErProcessor(id=1, server_node_id=self_server_node_id, processor_type=ProcessorTypes.EGG_PAIR, status=ProcessorStatus.RUNNING, command_endpoint=ErEndpoint("127.0.0.1", egg_ports[0]), transfer_endpoint=ErEndpoint("127.0.0.1", egg_transfer_ports[0])) roll = ErProcessor(id=1, server_node_id=self_server_node_id, processor_type=ProcessorTypes.ROLL_PAIR_MASTER, status=ProcessorStatus.RUNNING, command_endpoint=ErEndpoint("127.0.0.1", manager_port)) session = ErSession(session_id, processors=[egg, roll], options=options) context = RollPairContext(session) return context
def get_option(role, self_party_id=None, conf_file=default_props_file, deploy_mode=DeployModes.CLUSTER): print(f'conf file: {conf_file}') configs = configparser.ConfigParser() configs.read(conf_file) eggroll_configs = configs['eggroll'] options = {} if self_party_id is None: party_id = eggroll_configs[ RollSiteConfKeys.EGGROLL_ROLLSITE_PARTY_ID.key] else: party_id = self_party_id options['self_party_id'] = party_id options['self_role'] = role options['proxy_endpoint'] = \ ErEndpoint(host=eggroll_configs[RollSiteConfKeys.EGGROLL_ROLLSITE_HOST.key], port=int(eggroll_configs[RollSiteConfKeys.EGGROLL_ROLLSITE_PORT.key])) options[RollSiteConfKeys.EGGROLL_ROLLSITE_DEPLOY_MODE.key] = deploy_mode return options
def __init__(self, options: dict = None): if options is None: options = {} super().__init__(options) er_partition = options['er_partition'] self.partition = er_partition self.store_locator = er_partition._store_locator self.partition_id = er_partition._id self.namespace = self.store_locator._namespace #_store_type = StoreTypes.ROLLPAIR_ROLLSITE # self._store_locator = meta_pb2.StoreLocator(storeType=_store_type, # namespace=self.namespace, # name=self.store_locator._name, # partitioner=self.store_locator._partitioner, # serdes=self.store_locator._serdes, # totalPartitions=self.store_locator._total_partitions) self.roll_site_header_string = options.get('roll_site_header', None) self.is_writable = False if self.roll_site_header_string: self.roll_site_header = ErRollSiteHeader.from_proto_string( self.roll_site_header_string.encode(stringify_charset)) self.proxy_endpoint = ErEndpoint.from_proto_string( options['proxy_endpoint'].encode(stringify_charset)) self.obj_type = options['obj_type'] self.is_writable = True L.info( f"writable RollSiteAdapter: {self.namespace}, {self.partition_id}. proxy_endpoint: {self.proxy_endpoint}, partition: {self.partition}" )
def init_roll_site_context(runtime_conf, session_id): from eggroll.roll_site.roll_site import RollSiteContext from eggroll.roll_pair.roll_pair import RollPairContext LOGGER.info("init_roll_site_context runtime_conf: {}".format(runtime_conf)) session_instance = FateSession.get_instance()._eggroll.get_session() rp_context = RollPairContext(session_instance) role = runtime_conf.get("local").get("role") party_id = str(runtime_conf.get("local").get("party_id")) _path = file_utils.get_project_base_directory( ) + "/arch/conf/server_conf.json" server_conf = file_utils.load_json_conf(_path) host = server_conf.get('servers').get('proxy').get("host") port = server_conf.get('servers').get('proxy').get("port") options = { 'self_role': role, 'self_party_id': party_id, 'proxy_endpoint': ErEndpoint(host, int(port)) } rs_context = RollSiteContext(session_id, rp_ctx=rp_context, options=options) LOGGER.info("init_roll_site_context done: {}".format(rs_context.__dict__)) return rp_context, rs_context
def __init__(self, roll_site_session_id, rp_ctx: RollPairContext, options: dict = None): if options is None: options = {} self.roll_site_session_id = roll_site_session_id self.rp_ctx = rp_ctx self.push_session_enabled = RollSiteConfKeys.EGGROLL_ROLLSITE_PUSH_SESSION_ENABLED.get_with( options) if self.push_session_enabled: # create session for push roll_pair and object self._push_session = ErSession( session_id=roll_site_session_id + "_push", options=rp_ctx.get_session().get_all_options()) self._push_rp_ctx = RollPairContext(session=self._push_session) L.info( f"push_session={self._push_session.get_session_id()} enabled") def stop_push_session(): self._push_session.stop() else: self._push_session = None self._push_rp_ctx = None self.role = options["self_role"] self.party_id = str(options["self_party_id"]) self._options = options self._registered_comm_types = dict() self.register_comm_type('grpc', RollSiteGrpc) endpoint = options["proxy_endpoint"] if isinstance(endpoint, str): splitted = endpoint.split(':') self.proxy_endpoint = ErEndpoint(host=splitted[0].strip(), port=int(splitted[1].strip())) elif isinstance(endpoint, ErEndpoint): self.proxy_endpoint = endpoint else: raise ValueError("endpoint only support str and ErEndpoint type") self.is_standalone = RollSiteConfKeys.EGGROLL_ROLLSITE_DEPLOY_MODE.get_with( options) == "standalone" # if self.is_standalone: # self.stub = None # else: # channel = self.grpc_channel_factory.create_channel(self.proxy_endpoint) # self.stub = proxy_pb2_grpc.DataTransferServiceStub(channel) self.pushing_latch = CountDownLatch(0) self.rp_ctx.get_session().add_exit_task(self._wait_push_complete) if self.push_session_enabled: self.rp_ctx.get_session().add_exit_task(stop_push_session) self._wait_push_exit_timeout = int( RollSiteConfKeys.EGGROLL_ROLLSITE_PUSH_OVERALL_TIMEOUT_SEC. get_with(options)) L.info(f"inited RollSiteContext: {self.__dict__}")
def __init__(self, options: dict = None): if options is None: options = {} self.__endpoint = ErEndpoint( options[NodeManagerConfKeys.CONFKEY_NODE_MANAGER_HOST], int(options[NodeManagerConfKeys.CONFKEY_NODE_MANAGER_PORT])) if 'serdes_type' in options: self.__serdes_type = options['serdes_type'] else: self.__serdes_type = SerdesTypes.PROTOBUF self.__command_client = CommandClient()
def __init__(self, options: dict = None): if options is None: options = {} super().__init__(options) er_partition = options['er_partition'] self.partition = er_partition self.store_locator = er_partition._store_locator self.partition_id = er_partition._id self.namespace = self.store_locator._namespace #_store_type = StoreTypes.ROLLPAIR_ROLLSITE # self._store_locator = meta_pb2.StoreLocator(storeType=_store_type, # namespace=self.namespace, # name=self.store_locator._name, # partitioner=self.store_locator._partitioner, # serdes=self.store_locator._serdes, # totalPartitions=self.store_locator._total_partitions) self.roll_site_header_string = options.get('roll_site_header', None) self.is_writable = False if self.roll_site_header_string: self.roll_site_header = ErRollSiteHeader.from_proto_string( self.roll_site_header_string.encode(stringify_charset)) self.roll_site_header._options['partition_id'] = self.partition_id self.proxy_endpoint = ErEndpoint.from_proto_string( options['proxy_endpoint'].encode(stringify_charset)) self.obj_type = options['obj_type'] self.is_writable = True L.trace( f"writable RollSiteAdapter: {self.namespace}, partition_id={self.partition_id}. proxy_endpoint={self.proxy_endpoint}, partition={self.partition}" ) self.unarycall_max_retry_cnt = int( RollSiteConfKeys.EGGROLL_ROLLSITE_UNARYCALL_CLIENT_MAX_RETRY. get_with(options)) self.push_max_retry_cnt = int( RollSiteConfKeys.EGGROLL_ROLLSITE_PUSH_CLIENT_MAX_RETRY.get_with( options)) self.push_overall_timeout = int( RollSiteConfKeys.EGGROLL_ROLLSITE_OVERALL_TIMEOUT_SEC.get_with( options)) self.push_completion_wait_timeout = int( RollSiteConfKeys.EGGROLL_ROLLSITE_COMPLETION_WAIT_TIMEOUT_SEC. get_with(options)) self.push_packet_interval_timeout = int( RollSiteConfKeys.EGGROLL_ROLLSITE_PACKET_INTERVAL_TIMEOUT_SEC. get_with(options))
def get_option(role, conf_file=default_props_file): print(f'conf file: {conf_file}') configs = configparser.ConfigParser() configs.read(conf_file) eggroll_configs = configs['eggroll'] options = {} party_id = eggroll_configs[RollSiteConfKeys.EGGROLL_ROLLSITE_PARTY_ID.key] options['self_party_id'] = party_id options['self_role'] = role options['proxy_endpoint'] = \ ErEndpoint(host=eggroll_configs[RollSiteConfKeys.EGGROLL_ROLLSITE_HOST.key], port=int(eggroll_configs[RollSiteConfKeys.EGGROLL_ROLLSITE_PORT.key])) return options
def test_send(self): transfer_client = TransferClient() broker = FifoBroker() broker.put(b'hello') broker.put(b'world') broker.put(b'this') broker.put(b'is') broker.put(b'a') broker.put(b'test') broker.signal_write_finish() future = transfer_client.send(broker=broker, endpoint=ErEndpoint(host='localhost', port=transfer_port), tag='test') future.result()
def __init__(self, rp_ctx, rs_session_id, party, proxy_endpoint): LOGGER.debug(f"[federation.eggroll]init federation: " f"rp_session_id={rp_ctx.session_id}, rs_session_id={rs_session_id}, " f"party={party}, proxy_endpoint={proxy_endpoint}") from eggroll.core.meta_model import ErEndpoint if isinstance(proxy_endpoint, str): splited = proxy_endpoint.split(':') proxy_endpoint = ErEndpoint(host=splited[0].strip(), port=int(splited[1].strip())) options = { 'self_role': party.role, 'self_party_id': party.party_id, 'proxy_endpoint': proxy_endpoint } self._rsc = RollSiteContext(rs_session_id, rp_ctx=rp_ctx, options=options) LOGGER.debug(f"[federation.eggroll]init federation context done")
def __init__(self, roll_site_session_id, rp_ctx: RollPairContext, options: dict = None): if options is None: options = {} self.roll_site_session_id = roll_site_session_id self.rp_ctx = rp_ctx self.role = options["self_role"] self.party_id = str(options["self_party_id"]) self._options = options endpoint = options["proxy_endpoint"] if isinstance(endpoint, str): splitted = endpoint.split(':') self.proxy_endpoint = ErEndpoint(host=splitted[0].strip(), port=int(splitted[1].strip())) elif isinstance(endpoint, ErEndpoint): self.proxy_endpoint = endpoint else: raise ValueError("endpoint only support str and ErEndpoint type") self.is_standalone = RollSiteConfKeys.EGGROLL_ROLLSITE_DEPLOY_MODE.get_with( options) == "standalone" if self.is_standalone: self.stub = None else: channel = self.grpc_channel_factory.create_channel( self.proxy_endpoint) self.stub = proxy_pb2_grpc.DataTransferServiceStub(channel) self.pushing_latch = CountDownLatch(0) self.rp_ctx.get_session().add_exit_task(self._wait_push_complete) self._wait_push_exit_timeout = int( RollSiteConfKeys.EGGROLL_ROLLSITE_PUSH_OVERALL_TIMEOUT_SEC. get_with(options)) L.info(f"inited RollSiteContext: {self.__dict__}")
def __init__(self, options=None): if options is None: options = {} static_er_conf = get_static_er_conf() host = options.get( ClusterManagerConfKeys.CONFKEY_CLUSTER_MANAGER_HOST, static_er_conf.get( ClusterManagerConfKeys.CONFKEY_CLUSTER_MANAGER_HOST, None)) port = options.get( ClusterManagerConfKeys.CONFKEY_CLUSTER_MANAGER_PORT, static_er_conf.get( ClusterManagerConfKeys.CONFKEY_CLUSTER_MANAGER_PORT, None)) if not host or not port: raise ValueError( f'failed to load host or port in creating cluster manager client. host: {host}, port: {port}' ) self.__endpoint = ErEndpoint(host, int(port)) if 'serdes_type' in options: self.__serdes_type = options['serdes_type'] else: self.__serdes_type = SerdesTypes.PROTOBUF self.__command_client = CommandClient()
options['self_party_id'] = party_id options['self_role'] = role options['proxy_endpoint'] = \ ErEndpoint(host=eggroll_configs[RollSiteConfKeys.EGGROLL_ROLLSITE_HOST.key], port=int(eggroll_configs[RollSiteConfKeys.EGGROLL_ROLLSITE_PORT.key])) return options host_ip = 'localhost' guest_ip = 'localhost' host_options = { 'self_role': 'host', 'self_party_id': 10001, 'proxy_endpoint': ErEndpoint(host=host_ip, port=9395), } guest_options = { 'self_role': 'guest', 'self_party_id': 10002, 'proxy_endpoint': ErEndpoint(host=guest_ip, port=9396), } ER_STORE1 = ErStore(store_locator=ErStoreLocator( store_type=StoreTypes.ROLLPAIR_LEVELDB, namespace="namespace", name="name")) roll_site_session_id = f'atest'
def serve(args): prefix = 'v1/egg-pair' set_data_dir(args.data_dir) CommandRouter.get_instance().register( service_name=f"{prefix}/runTask", route_to_module_name="eggroll.roll_pair.egg_pair", route_to_class_name="EggPair", route_to_method_name="run_task") max_workers = int( RollPairConfKeys. EGGROLL_ROLLPAIR_EGGPAIR_SERVER_EXECUTOR_POOL_MAX_SIZE.get()) executor_pool_type = CoreConfKeys.EGGROLL_CORE_DEFAULT_EXECUTOR_POOL.get() command_server = grpc.server( create_executor_pool(canonical_name=executor_pool_type, max_workers=max_workers, thread_name_prefix="eggpair-command-server"), options= [("grpc.max_metadata_size", int(CoreConfKeys. EGGROLL_CORE_GRPC_SERVER_CHANNEL_MAX_INBOUND_METADATA_SIZE.get()) ), ('grpc.max_send_message_length', int(CoreConfKeys. EGGROLL_CORE_GRPC_SERVER_CHANNEL_MAX_INBOUND_MESSAGE_SIZE.get()) ), ('grpc.max_receive_message_length', int(CoreConfKeys. EGGROLL_CORE_GRPC_SERVER_CHANNEL_MAX_INBOUND_MESSAGE_SIZE.get()) ), ('grpc.keepalive_time_ms', int(CoreConfKeys.CONFKEY_CORE_GRPC_CHANNEL_KEEPALIVE_TIME_SEC.get()) * 1000), ('grpc.keepalive_timeout_ms', int(CoreConfKeys. CONFKEY_CORE_GRPC_SERVER_CHANNEL_KEEPALIVE_TIMEOUT_SEC.get()) * 1000), ('grpc.keepalive_permit_without_calls', int(CoreConfKeys. CONFKEY_CORE_GRPC_SERVER_CHANNEL_KEEPALIVE_WITHOUT_CALLS_ENABLED. get())), ('grpc.per_rpc_retry_buffer_size', int(CoreConfKeys.CONFKEY_CORE_GRPC_SERVER_CHANNEL_RETRY_BUFFER_SIZE. get())), ('grpc.so_reuseport', False)]) command_servicer = CommandServicer() command_pb2_grpc.add_CommandServiceServicer_to_server( command_servicer, command_server) transfer_servicer = GrpcTransferServicer() port = args.port transfer_port = args.transfer_port port = command_server.add_insecure_port(f'[::]:{port}') if transfer_port == "-1": transfer_server = command_server transfer_port = port transfer_pb2_grpc.add_TransferServiceServicer_to_server( transfer_servicer, transfer_server) else: transfer_server_max_workers = int( RollPairConfKeys. EGGROLL_ROLLPAIR_EGGPAIR_DATA_SERVER_EXECUTOR_POOL_MAX_SIZE.get()) transfer_server = grpc.server( create_executor_pool(canonical_name=executor_pool_type, max_workers=transfer_server_max_workers, thread_name_prefix="transfer_server"), options= [('grpc.max_metadata_size', int(CoreConfKeys. EGGROLL_CORE_GRPC_SERVER_CHANNEL_MAX_INBOUND_METADATA_SIZE. get())), ('grpc.max_send_message_length', int(CoreConfKeys. EGGROLL_CORE_GRPC_SERVER_CHANNEL_MAX_INBOUND_MESSAGE_SIZE. get())), ('grpc.max_receive_message_length', int(CoreConfKeys. EGGROLL_CORE_GRPC_SERVER_CHANNEL_MAX_INBOUND_MESSAGE_SIZE. get())), ('grpc.keepalive_time_ms', int(CoreConfKeys. CONFKEY_CORE_GRPC_SERVER_CHANNEL_KEEPALIVE_WITHOUT_CALLS_ENABLED .get()) * 1000), ('grpc.keepalive_timeout_ms', int(CoreConfKeys. CONFKEY_CORE_GRPC_SERVER_CHANNEL_KEEPALIVE_TIMEOUT_SEC.get()) * 1000), ('grpc.keepalive_permit_without_calls', int(CoreConfKeys. CONFKEY_CORE_GRPC_SERVER_CHANNEL_KEEPALIVE_WITHOUT_CALLS_ENABLED .get())), ('grpc.per_rpc_retry_buffer_size', int(CoreConfKeys. CONFKEY_CORE_GRPC_SERVER_CHANNEL_RETRY_BUFFER_SIZE.get())), ('grpc.so_reuseport', False)]) transfer_port = transfer_server.add_insecure_port( f'[::]:{transfer_port}') transfer_pb2_grpc.add_TransferServiceServicer_to_server( transfer_servicer, transfer_server) transfer_server.start() pid = os.getpid() L.info( f"starting egg_pair service, port: {port}, transfer port: {transfer_port}, pid: {pid}" ) command_server.start() cluster_manager = args.cluster_manager myself = None cluster_manager_client = None if cluster_manager: session_id = args.session_id server_node_id = int(args.server_node_id) static_er_conf = get_static_er_conf() static_er_conf['server_node_id'] = server_node_id if not session_id: raise ValueError('session id is missing') options = {SessionConfKeys.CONFKEY_SESSION_ID: args.session_id} myself = ErProcessor(id=int(args.processor_id), server_node_id=server_node_id, processor_type=ProcessorTypes.EGG_PAIR, command_endpoint=ErEndpoint(host='localhost', port=port), transfer_endpoint=ErEndpoint(host='localhost', port=transfer_port), pid=pid, options=options, status=ProcessorStatus.RUNNING) cluster_manager_host, cluster_manager_port = cluster_manager.strip( ).split(':') L.info(f'egg_pair cluster_manager: {cluster_manager}') cluster_manager_client = ClusterManagerClient( options={ ClusterManagerConfKeys.CONFKEY_CLUSTER_MANAGER_HOST: cluster_manager_host, ClusterManagerConfKeys.CONFKEY_CLUSTER_MANAGER_PORT: cluster_manager_port }) cluster_manager_client.heartbeat(myself) if platform.system() == "Windows": t1 = threading.Thread(target=stop_processor, args=[cluster_manager_client, myself]) t1.start() L.info(f'egg_pair started at port={port}, transfer_port={transfer_port}') run = True def exit_gracefully(signum, frame): nonlocal run run = False L.info( f'egg_pair {args.processor_id} at port={port}, transfer_port={transfer_port}, pid={pid} receives signum={signal.getsignal(signum)}, stopping gracefully.' ) signal.signal(signal.SIGTERM, exit_gracefully) signal.signal(signal.SIGINT, exit_gracefully) while run: time.sleep(1) L.info(f'sending exit heartbeat to cm') if cluster_manager: myself._status = ProcessorStatus.STOPPED cluster_manager_client.heartbeat(myself) GrpcChannelFactory.shutdown_all_now() L.info(f'closing RocksDB open dbs') #todo:1: move to RocksdbAdapter and provide a cleanup method from eggroll.core.pair_store.rocksdb import RocksdbAdapter for path, db in RocksdbAdapter.db_dict.items(): del db gc.collect() L.info(f'system metric at exit: {get_system_metric(1)}') L.info( f'egg_pair {args.processor_id} at port={port}, transfer_port={transfer_port}, pid={pid} stopped gracefully' )
# See the License for the specific language governing permissions and # limitations under the License. import queue import threading import unittest from concurrent.futures import ThreadPoolExecutor from eggroll.core.conf_keys import TransferConfKeys from eggroll.core.datastructure.broker import FifoBroker, BrokerClosed from eggroll.core.meta_model import ErEndpoint from eggroll.core.transfer.transfer_service import TransferService, \ GrpcTransferService, TransferClient transfer_port = 20002 transfer_endpont = ErEndpoint('localhost', transfer_port) class TestTransfer(unittest.TestCase): def setUp(self) -> None: self.__executor_pool = ThreadPoolExecutor(max_workers=5) def test_recv(self): def start_server(): transfer_service = GrpcTransferService() options = { TransferConfKeys.CONFKEY_TRANSFER_SERVICE_PORT: transfer_port } transfer_service.start(options=options)
def serve(args): prefix = 'v1/egg-pair' set_data_dir(args.data_dir) CommandRouter.get_instance().register( service_name=f"{prefix}/runTask", route_to_module_name="eggroll.roll_pair.egg_pair", route_to_class_name="EggPair", route_to_method_name="run_task") command_server = grpc.server( futures.ThreadPoolExecutor(max_workers=500, thread_name_prefix="grpc_server"), options=[("grpc.max_metadata_size", 32 << 20), (cygrpc.ChannelArgKey.max_send_message_length, 2 << 30 - 1), (cygrpc.ChannelArgKey.max_receive_message_length, 2 << 30 - 1) ]) command_servicer = CommandServicer() command_pb2_grpc.add_CommandServiceServicer_to_server( command_servicer, command_server) transfer_servicer = GrpcTransferServicer() port = args.port transfer_port = args.transfer_port port = command_server.add_insecure_port(f'[::]:{port}') if transfer_port == "-1": transfer_server = command_server transfer_port = port transfer_pb2_grpc.add_TransferServiceServicer_to_server( transfer_servicer, transfer_server) else: transfer_server = grpc.server( futures.ThreadPoolExecutor(max_workers=500, thread_name_prefix="transfer_server"), options=[ (cygrpc.ChannelArgKey.max_send_message_length, 2 << 30 - 1), (cygrpc.ChannelArgKey.max_receive_message_length, 2 << 30 - 1), ('grpc.max_metadata_size', 32 << 20) ]) transfer_port = transfer_server.add_insecure_port( f'[::]:{transfer_port}') transfer_pb2_grpc.add_TransferServiceServicer_to_server( transfer_servicer, transfer_server) transfer_server.start() L.info( f"starting egg_pair service, port:{port}, transfer port: {transfer_port}" ) command_server.start() cluster_manager = args.cluster_manager myself = None cluster_manager_client = None if cluster_manager: session_id = args.session_id if not session_id: raise ValueError('session id is missing') options = {SessionConfKeys.CONFKEY_SESSION_ID: args.session_id} myself = ErProcessor(id=int(args.processor_id), server_node_id=int(args.server_node_id), processor_type=ProcessorTypes.EGG_PAIR, command_endpoint=ErEndpoint(host='localhost', port=port), transfer_endpoint=ErEndpoint(host='localhost', port=transfer_port), pid=os.getpid(), options=options, status=ProcessorStatus.RUNNING) cluster_manager_host, cluster_manager_port = cluster_manager.strip( ).split(':') L.info(f'cluster_manager: {cluster_manager}') cluster_manager_client = ClusterManagerClient( options={ ClusterManagerConfKeys.CONFKEY_CLUSTER_MANAGER_HOST: cluster_manager_host, ClusterManagerConfKeys.CONFKEY_CLUSTER_MANAGER_PORT: cluster_manager_port }) cluster_manager_client.heartbeat(myself) L.info(f'egg_pair started at port {port}, transfer_port {transfer_port}') run = True def exit_gracefully(signum, frame): nonlocal run run = False signal.signal(signal.SIGTERM, exit_gracefully) signal.signal(signal.SIGINT, exit_gracefully) import time while run: time.sleep(1) if cluster_manager: myself._status = ProcessorStatus.STOPPED cluster_manager_client.heartbeat(myself) L.info( f'egg_pair at port {port}, transfer_port {transfer_port} stopped gracefully' )