def __init__(self, g_pool, node_name=None, sync_group_prefix='default', base_bias=1.):
super().__init__(g_pool)
self.sync_group_prefix = sync_group_prefix
self.discovery = None
self.leaderboard = []
self.has_been_master = 0.
self.has_been_synced = 0.
self.tie_breaker = random.random()
self.base_bias = base_bias
self.master_service = Clock_Sync_Master(self.g_pool.get_timestamp)
self.follower_service = None # only set if there is a better server than us
self.restart_discovery(node_name)
def run_time_sync_master(group):
pts_group = group + '-time_sync-v1'
# the time source in the example is python time.time you can change this.
# replace with an implementation that give your custom time in floating sec.
clock_service = Clock_Sync_Master(time)
# This example is a clock service only, not a clock follower.
# Therefore the rank is designed to always trump all others.
rank = 1000
discovery = Pyre('pupil-helper-service')
discovery.join(pts_group)
discovery.start()
logger.info('Joining "{}" group with rank {}'.format(pts_group, rank))
def announce_clock_service_info():
discovery.shout(
pts_group,
[repr(rank).encode(),
repr(clock_service.port).encode()])
try:
for event in discovery.events():
if event.type == 'JOIN' and event.group == pts_group:
logger.info(
'"{}" joined "{}" group. Announcing service.'.format(
event.peer_name, pts_group))
announce_clock_service_info()
except KeyboardInterrupt:
pass
finally:
logger.info('Leaving "{}" group'.format(pts_group))
discovery.leave(pts_group)
discovery.stop()
clock_service.stop()
def thread_loop(self, context, pipe):
n = Pyre(self.name)
n.join(self.group)
n.start()
poller = zmq.Poller()
poller.register(pipe, zmq.POLLIN)
poller.register(n.socket(), zmq.POLLIN)
front, back = zhelper.zcreate_pipe(context)
poller.register(back, zmq.POLLIN)
def wake_up():
#on app close this timer calls a closed socket. We simply catch it here.
try:
front.send('wake_up')
except Exception as e:
logger.debug('Orphaned timer thread raised error: %s' % e)
t = Timer(self.time_sync_announce_interval, wake_up)
t.daemon = True
t.start()
while (True):
try:
#this should not fail but it does sometimes. We need to clean this out.
# I think we are not treating sockets correclty as they are not thread-save.
items = dict(poller.poll())
except zmq.ZMQError:
logger.warning('Socket fail.')
continue
if back in items and items[back] == zmq.POLLIN:
back.recv()
#timeout events are used for pupil sync.
#annouce masterhood every interval time:
if isinstance(self.time_sync_node, Clock_Sync_Master):
n.shouts(
self.group, SYNC_TIME_MASTER_ANNOUNCE +
"%s" % self.clock_master_worthiness() + msg_delimeter +
'%s' % self.time_sync_node.port)
# synced slave: see if we should become master if we dont hear annoncement within time.
elif isinstance(self.time_sync_node, Clock_Sync_Follower
) and not self.time_sync_node.offset_remains:
if self.get_unadjusted_time(
) - self.last_master_announce > self.time_sync_wait_interval_short:
self.time_sync_node.terminate()
self.time_sync_node = Clock_Sync_Master(
time_fn=self.get_time)
n.shouts(
self.group, SYNC_TIME_MASTER_ANNOUNCE +
"%s" % self.clock_master_worthiness() +
msg_delimeter + '%s' % self.time_sync_node.port)
# unsynced slave or none should wait longer but eventually take over
elif self.get_unadjusted_time(
) - self.last_master_announce > self.time_sync_wait_interval_long:
if self.time_sync_node:
self.time_sync_node.terminate()
self.time_sync_node = Clock_Sync_Master(
time_fn=self.get_time)
n.shouts(
self.group, SYNC_TIME_MASTER_ANNOUNCE +
"%s" % self.clock_master_worthiness() + msg_delimeter +
'%s' % self.time_sync_node.port)
t = Timer(self.time_sync_announce_interval, wake_up)
t.daemon = True
t.start()
if pipe in items and items[pipe] == zmq.POLLIN:
message = pipe.recv()
# message to quit
if message.decode('utf-8') == exit_thread:
break
else:
logger.debug("Shout '%s' to '%s' " % (message, self.group))
n.shouts(self.group, message)
if n.socket() in items and items[n.socket()] == zmq.POLLIN:
cmds = n.recv()
msg_type = cmds.pop(0)
msg_type = msg_type.decode('utf-8')
if msg_type == "SHOUT":
uuid, name, group, msg = cmds
logger.debug("'%s' shouts '%s'." % (name, msg))
self._handle_msg(uuid, name, msg, n)
elif msg_type == "WHISPER":
uuid, name, msg = cmds
logger.debug("'%s/' whispers '%s'." % (name, msg))
self._handle_msg_whisper(uuid, name, msg, n)
elif msg_type == "JOIN":
uuid, name, group = cmds
if group == self.group:
self.group_members[uuid] = name
self.update_gui()
elif msg_type == "EXIT":
uuid, name = cmds
try:
del self.group_members[uuid]
except KeyError:
pass
else:
self.update_gui()
# elif msg_type == "LEAVE":
# uuid,name,group = cmds
# elif msg_type == "ENTER":
# uuid,name,headers,ip = cmds
# logger.warning((uuid,'name',headers,ip))
else:
pass
logger.debug('thread_loop closing.')
self.thread_pipe = None
n.stop()
def on_notify(self, notification):
"""Synchronize time of Actors across local network.
The notification scheme is used to handle interal timing
and to talk to remote pers via the `Pupil_Groups` plugin.
Reacts to notifications:
``time_sync.master_announcement``: React accordingly to annouce notification from remote peer.
``time_sync.master_announce_interval``: Re-annouce clock masterhood.
``time_sync.master_announce_timeout``: React accordingly when no master announcement has appeard whithin timeout.
Emits notifications:
``time_sync.master_announcement``: Announce masterhood to remote peers (remote notification).
``time_sync.master_announce_interval``: Re-announce masterhood reminder (delayed notification).
``time_sync.master_announce_timeout``: Timeout for foreind master announcement (delayed notification).
"""
if notification['subject'].startswith('time_sync.master_announcement'):
if self.is_master:
if notification['worthiness'] > self.clock_master_worthiness():
#We need to yield.
self.time_sync_node.stop()
self.time_sync_node = None
else:
#Denounce the lesser competition.
n = {
'subject': 'time_sync.master_announcement',
'host': self.time_sync_node.host,
'port': self.time_sync_node.port,
'worthiness': self.clock_master_worthiness(),
'remote_notify': 'all'
}
self.notify_all(n)
if self.is_follower:
# update follower info
self.time_sync_node.host = notification['host']
self.time_sync_node.port = notification['port']
if self.is_nothing:
# Create follower.
logger.debug("Clock will sync with {}".format(
notification['host']))
self.time_sync_node = Clock_Sync_Follower(
notification['host'],
port=notification['port'],
interval=10,
time_fn=self.get_time,
jump_fn=self.jump_time,
slew_fn=self.slew_time)
if not self.is_master:
#(Re)set the timer.
self.notify_all(self.master_announce_timeout_notification)
elif notification['subject'].startswith(
'time_sync.master_announce_timeout'):
if self.is_master:
pass
else:
#We have not heard from a master in too long.
logger.info("Elevate self to clock master.")
self.time_sync_node = Clock_Sync_Master(
self.g_pool.get_timestamp)
n = {
'subject': 'time_sync.master_announcement',
'host': self.time_sync_node.host,
'port': self.time_sync_node.port,
'worthiness': self.clock_master_worthiness(),
'remote_notify': 'all'
}
self.notify_all(n)
self.notify_all(self.master_announce_interval_notification)
elif notification['subject'].startswith(
'time_sync.master_announce_interval'):
# The time has come to remind others of our master hood.
if self.is_master:
n = {
'subject': 'time_sync.master_announcement',
'host': self.time_sync_node.host,
'port': self.time_sync_node.port,
'worthiness': self.clock_master_worthiness(),
'remote_notify': 'all'
}
self.notify_all(n)
# Set the next annouce timer.
self.notify_all(self.master_announce_interval_notification)
