def start_heartbeat(self):
while self.running:
try:
# do heartbeat
self.storage.clear_dead_members(self.ttl_ms)
self.storage.update_member(self.server_uri, self.uuid)
self.living_members = self.storage.list_living_members(
self.ttl_ms)
if not self.notified_others_after_start:
for member in self.living_members:
if member.server_uri == self.server_uri:
continue
channel = grpc.insecure_channel(member.server_uri)
self.member_connections[member.server_uri] = \
notification_service_pb2_grpc.NotificationServiceStub(channel)
try:
self.member_connections[
member.server_uri].notifyNewMember(
NotifyNewMemberRequest(
member=member_to_proto(
Member(
1, self.server_uri,
int(time.time_ns() /
1000000)))))
except grpc.RpcError:
logging.error("Notify new member to '%s' failed." %
member.server_uri,
exc_info=True)
self.notified_others_after_start = True
except Exception as e:
logging.error(
"Exception thrown when send heartbeat to the HA storage.",
exc_info=True)
sleep_and_detecting_running(self.ttl_ms / 2, lambda: self.running)
def start_notify(self):
while self.running:
# update connections
living_member_server_uris = set()
for member in self.living_members:
if member.server_uri == self.server_uri:
continue
if member.server_uri not in self.member_connections:
try:
channel = grpc.insecure_channel(member.server_uri)
self.member_connections[member.server_uri] = \
notification_service_pb2_grpc.NotificationServiceStub(channel)
except Exception:
logging.error(
"Exception thrown when connect to another member: %s."
% member.server_uri,
exc_info=True)
continue
living_member_server_uris.add(member.server_uri)
for server_uri in list(self.member_connections.keys()):
if server_uri not in living_member_server_uris:
del self.member_connections[server_uri]
notifies = []
while len(self.cached_notify) > 0:
event = self.cached_notify.popleft()
notify = Notify(key=event.key, namespace=event.namespace)
notifies.append(notify)
# notify others
if len(notifies) > 0:
for server_uri, stub in self.member_connections.items():
try:
stub.notify(NotifyRequest(notifies=notifies))
except Exception:
logging.error(
"Exception thrown when notify another member: %s."
% server_uri,
exc_info=True)
sleep_and_detecting_running(self.min_notify_interval_ms,
lambda: self.running, 100)
def __init__(self, server_uri):
channel = grpc.insecure_channel(server_uri)
self.notification_stub = notification_service_pb2_grpc.NotificationServiceStub(channel)
self.threads = {}
self.lock = threading.Lock()
def __init__(self,
server_uri: str,
default_namespace: str = None,
enable_ha: bool = False,
list_member_interval_ms: int = 5000,
retry_interval_ms: int = 1000,
retry_timeout_ms: int = 10000,
sender: str = None):
"""
The constructor of the NotificationClient.
:param server_uri: Target server uri/uris. If `enable_ha` is True, multiple uris separated
by "," can be accepted.
:param default_namespace: The default namespace that this client is working on.
:param enable_ha: Enable high-available functionality.
:param list_member_interval_ms: When `enable_ha` is True, this client will request the
living members periodically. A member means a server node
of the Notification server cluster. This param specifies
the interval of the listing member requests.
:param retry_interval_ms: When `enable_ha` is True and a rpc call has failed on all the
living members, this client will retry until success or timeout.
This param specifies the retry interval.
:param retry_timeout_ms: When `enable_ha` is True and a rpc call has failed on all the
living members, this client will retry until success or timeout.
This param specifies the retry timeout.
:param sender: The identify of the client.
"""
channel = grpc.insecure_channel(server_uri)
self._default_namespace = default_namespace
self.notification_stub = notification_service_pb2_grpc.NotificationServiceStub(
channel)
self.threads = {} # type: Dict[Any, List[threading.Thread]]
self.lock = threading.Lock()
self.enable_ha = enable_ha
self.list_member_interval_ms = list_member_interval_ms
self.retry_interval_ms = retry_interval_ms
self.retry_timeout_ms = retry_timeout_ms
self._sender = sender
if self.enable_ha:
server_uris = server_uri.split(",")
self.living_members = []
self.current_uri = None
last_error = None
for server_uri in server_uris:
channel = grpc.insecure_channel(server_uri)
notification_stub = notification_service_pb2_grpc.NotificationServiceStub(
channel)
try:
request = ListMembersRequest(timeout_seconds=0)
response = notification_stub.listMembers(request)
if response.return_code == ReturnStatus.SUCCESS:
self.living_members = [
proto_to_member(proto).server_uri
for proto in response.members
]
else:
raise Exception(response.return_msg)
self.current_uri = server_uri
self.notification_stub = notification_stub
break
except grpc.RpcError as e:
last_error = e
if self.current_uri is None:
raise Exception("No available server uri!") from last_error
self.ha_change_lock = threading.Lock()
self.ha_running = True
self.notification_stub = self._wrap_rpcs(self.notification_stub,
server_uri)
self.list_member_thread = threading.Thread(
target=self._list_members, daemon=True)
self.list_member_thread.start()