def run_time_sync_master(group):
pts_group = group + '-time_sync-v1'
# the time source in the example is python time.time you can change this.
# replace with an implementation that give your custom time in floating sec.
clock_service = Clock_Sync_Master(time)
# This example is a clock service only, not a clock follower.
# Therefore the rank is designed to always trump all others.
rank = 1000
discovery = Pyre('pupil-helper-service')
discovery.join(pts_group)
discovery.start()
logger.info('Joining "{}" group with rank {}'.format(pts_group, rank))
def announce_clock_service_info():
discovery.shout(pts_group, [repr(rank).encode(), repr(clock_service.port).encode()])
try:
for event in discovery.events():
if event.type == 'JOIN' and event.group == pts_group:
logger.info('"{}" joined "{}" group. Announcing service.'.format(event.peer_name, pts_group))
announce_clock_service_info()
except KeyboardInterrupt:
pass
finally:
logger.info('Leaving "{}" group'.format(pts_group))
discovery.leave(pts_group)
discovery.stop()
clock_service.stop()
def __init__(self, g_pool, node_name=None, sync_group_prefix='default', base_bias=1.):
super().__init__(g_pool)
self.sync_group_prefix = sync_group_prefix
self.discovery = None
self.leaderboard = []
self.has_been_master = 0.
self.has_been_synced = 0.
self.tie_breaker = random.random()
self.base_bias = base_bias
self.master_service = Clock_Sync_Master(self.g_pool.get_timestamp)
self.follower_service = None # only set if there is a better server than us
self.restart_discovery(node_name)
def run_time_sync_master(group):
pts_group = group + '-time_sync-v1'
# the time source in the example is python time.time you can change this.
# replace with an implementation that give your custom time in floating sec.
clock_service = Clock_Sync_Master(time)
# This example is a clock service only, not a clock follower.
# Therefore the rank is designed to always trump all others.
rank = 1000
discovery = Pyre('pupil-helper-service')
discovery.join(pts_group)
discovery.start()
logger.info('Joining "{}" group with rank {}'.format(pts_group, rank))
def announce_clock_service_info():
discovery.shout(
pts_group,
[repr(rank).encode(),
repr(clock_service.port).encode()])
try:
for event in discovery.events():
if event.type == 'JOIN' and event.group == pts_group:
logger.info(
'"{}" joined "{}" group. Announcing service.'.format(
event.peer_name, pts_group))
announce_clock_service_info()
except KeyboardInterrupt:
pass
finally:
logger.info('Leaving "{}" group'.format(pts_group))
discovery.leave(pts_group)
discovery.stop()
clock_service.stop()
def __init__(self, g_pool, node_name=None, sync_group_prefix='default', base_bias=1.):
super().__init__(g_pool)
self.menu = None
self.sync_group_prefix = sync_group_prefix
self.discovery = None
self.leaderboard = []
self.has_been_master = 0.
self.has_been_synced = 0.
self.tie_breaker = random.random()
self.base_bias = base_bias
self.master_service = Clock_Sync_Master(self.g_pool.get_timestamp)
self.follower_service = None # only set if there is a better server than us
self.restart_discovery(node_name)
class Time_Sync(Plugin):
"""Synchronize time across local network.
Implements the Pupil Time Sync protocol.
Acts as clock service and as follower if required.
See `time_sync_spec.md` for details.
"""
icon_chr = chr(0xEC15)
icon_font = "pupil_icons"
def __init__(self,
g_pool,
node_name=None,
sync_group_prefix="default",
base_bias=1.0):
super().__init__(g_pool)
self.sync_group_prefix = sync_group_prefix
self.discovery = None
self.leaderboard = []
self.has_been_master = 0.0
self.has_been_synced = 0.0
self.tie_breaker = random.random()
self.base_bias = base_bias
self.sync_group_members = {}
self.master_service = Clock_Sync_Master(self.g_pool.get_timestamp)
self.follower_service = None # only set if there is a better server than us
self.restart_discovery(node_name)
@property
def sync_group(self):
return self.sync_group_prefix + "-time_sync-" + __protocol_version__
@sync_group.setter
def sync_group(self, full_name):
self.sync_group_prefix = full_name.rsplit("-time_sync-" +
__protocol_version__,
maxsplit=1)[0]
def init_ui(self):
self.add_menu()
self.menu.label = "Network Time Sync"
help_str = "Synchonize time of Pupil Captures across the local network."
self.menu.append(
ui.Info_Text("Protocol version: " + __protocol_version__))
self.menu.append(ui.Info_Text(help_str))
help_str = "All pupil nodes of one group share a Master clock."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(
ui.Text_Input("node_name",
self,
label="Node Name",
setter=self.restart_discovery))
self.menu.append(
ui.Text_Input(
"sync_group_prefix",
self,
label="Sync Group",
setter=self.change_sync_group,
))
def sync_status():
if self.follower_service:
return str(self.follower_service)
else:
return "Clock Master"
self.menu.append(
ui.Text_Input("sync status",
getter=sync_status,
setter=lambda _: _,
label="Status"))
def set_bias(bias):
if bias < 0:
bias = 0.0
self.base_bias = bias
self.announce_clock_master_info()
self.evaluate_leaderboard()
help_str = "The clock service with the highest bias becomes clock master."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(
ui.Text_Input("base_bias",
self,
label="Master Bias",
setter=set_bias))
self.menu.append(
ui.Text_Input("leaderboard", self, label="Master Nodes in Group"))
self.sync_group_members_menu = ui.Growing_Menu("Sync Group Members")
self.menu.append(self.sync_group_members_menu)
def recent_events(self, events):
should_announce = False
for evt in self.discovery.recent_events():
if evt.type == "SHOUT":
try:
self.update_leaderboard(evt.peer_uuid, evt.peer_name,
float(evt.msg[0]), int(evt.msg[1]))
except Exception as e:
logger.debug("Garbage raised `{}` -- dropping.".format(e))
self.evaluate_leaderboard()
elif evt.type == "JOIN" and evt.group == self.sync_group:
should_announce = True
self.insert_sync_group_member(evt.peer_uuid, evt.peer_name)
elif (evt.type == "LEAVE"
and evt.group == self.sync_group) or evt.type == "EXIT":
self.remove_from_leaderboard(evt.peer_uuid)
self.evaluate_leaderboard()
self.remove_sync_group_member(evt.peer_uuid)
if should_announce:
self.announce_clock_master_info()
if (not self.has_been_synced and self.follower_service
and self.follower_service.in_sync):
self.has_been_synced = 1.0
self.announce_clock_master_info()
self.evaluate_leaderboard()
def update_leaderboard(self, uuid, name, rank, port):
for cs in self.leaderboard:
if cs.uuid == uuid:
if (cs.rank != rank) or (cs.port != port):
self.remove_from_leaderboard(cs.uuid)
break
else:
# no changes. Just leave as is
return
# clock service was not encountered before or has changed adding it to leaderboard
cs = Clock_Service(uuid, name, rank, port)
heappush(self.leaderboard, cs)
logger.debug("{} added".format(cs))
def remove_from_leaderboard(self, uuid):
for cs in self.leaderboard:
if cs.uuid == uuid:
self.leaderboard.remove(cs)
logger.debug("{} removed".format(cs))
break
def evaluate_leaderboard(self):
if not self.leaderboard:
logger.debug("nobody on the leader board.")
return
current_leader = self.leaderboard[0]
if self.discovery.uuid() != current_leader.uuid:
# we are not the leader!
leader_ep = self.discovery.peer_address(current_leader.uuid)
leader_addr = urlparse(leader_ep).netloc.split(":")[0]
if self.follower_service is None:
# make new follower
self.follower_service = Clock_Sync_Follower(
leader_addr,
port=current_leader.port,
interval=10,
time_fn=self.get_time,
jump_fn=self.jump_time,
slew_fn=self.slew_time,
)
else:
# update follower_service
self.follower_service.host = leader_addr
self.follower_service.port = current_leader.port
return
# we are the leader
logger.debug("we are the leader")
if self.follower_service is not None:
self.follower_service.terminate()
self.follower_service = None
if not self.has_been_master:
self.has_been_master = 1.0
logger.debug("Become clock master with rank {}".format(self.rank))
self.announce_clock_master_info()
def insert_sync_group_member(self, uuid, name):
member_text = ui.Info_Text(name)
self.sync_group_members[uuid] = member_text
self.sync_group_members_menu.append(member_text)
self.sync_group_members_menu.elements.sort(
key=lambda text_field: text_field.text)
def insert_all_sync_group_members_from_group(self, group):
for uuid in self.discovery.peers_by_group(group):
name = self.discovery.get_peer_name(uuid)
self.insert_sync_group_member(uuid, name)
def remove_all_sync_group_members(self):
for uuid in list(self.sync_group_members.keys()):
self.remove_sync_group_member(uuid)
def remove_sync_group_member(self, uuid):
try:
self.sync_group_members_menu.remove(self.sync_group_members[uuid])
del self.sync_group_members[uuid]
except KeyError:
logger.debug("Peer has already been removed from members list.")
def announce_clock_master_info(self):
self.discovery.shout(
self.sync_group,
[
repr(self.rank).encode(),
repr(self.master_service.port).encode()
],
)
self.update_leaderboard(self.discovery.uuid(), self.node_name,
self.rank, self.master_service.port)
@property
def rank(self):
return (4 * self.base_bias + 2 * self.has_been_master +
self.has_been_synced + self.tie_breaker)
def get_time(self):
return self.g_pool.get_timestamp()
def slew_time(self, offset):
self.g_pool.timebase.value += offset
def jump_time(self, offset):
ok_to_change = True
for p in self.g_pool.plugins:
if p.class_name == "Recorder":
if p.running:
ok_to_change = False
logger.error(
"Request to change timebase during recording ignored. Turn off recording first."
)
break
if ok_to_change:
self.slew_time(offset)
logger.info(
"Pupil Sync has adjusted the clock by {}s".format(offset))
return True
else:
return False
def restart_discovery(self, name):
if self.discovery:
if self.discovery.name() == name:
return
else:
self.remove_all_sync_group_members()
self.discovery.leave(self.sync_group)
self.discovery.stop()
self.leaderboard = []
self.node_name = name or gethostname()
self.discovery = Pyre(self.node_name)
# Either joining network for the first time or rejoining the same group.
self.discovery.join(self.sync_group)
self.discovery.start()
self.announce_clock_master_info()
def change_sync_group(self, new_group_prefix):
if new_group_prefix != self.sync_group_prefix:
self.remove_all_sync_group_members()
self.discovery.leave(self.sync_group)
self.leaderboard = []
if self.follower_service:
self.follower_service.terminate()
self.follower = None
self.sync_group_prefix = new_group_prefix
self.discovery.join(self.sync_group)
self.insert_all_sync_group_members_from_group(self.sync_group)
self.announce_clock_master_info()
def deinit_ui(self):
for uuid in list(self.sync_group_members.keys()):
self.remove_sync_group_member(uuid)
self.remove_menu()
def get_init_dict(self):
return {
"node_name": self.node_name,
"sync_group_prefix": self.sync_group_prefix,
"base_bias": self.base_bias,
}
def cleanup(self):
self.discovery.leave(self.sync_group)
self.discovery.stop()
self.master_service.stop()
if self.follower_service:
self.follower_service.stop()
self.master_service = None
self.follower_service = None
self.discovery = None
def thread_loop(self, context, pipe):
n = Pyre(self.name)
n.join(self.group)
n.start()
poller = zmq.Poller()
poller.register(pipe, zmq.POLLIN)
poller.register(n.socket(), zmq.POLLIN)
front, back = zhelper.zcreate_pipe(context)
poller.register(back, zmq.POLLIN)
def wake_up():
#on app close this timer calls a closed socket. We simply catch it here.
try:
front.send('wake_up')
except Exception as e:
logger.debug('Orphaned timer thread raised error: %s' % e)
t = Timer(self.time_sync_announce_interval, wake_up)
t.daemon = True
t.start()
while (True):
try:
#this should not fail but it does sometimes. We need to clean this out.
# I think we are not treating sockets correclty as they are not thread-save.
items = dict(poller.poll())
except zmq.ZMQError:
logger.warning('Socket fail.')
continue
if back in items and items[back] == zmq.POLLIN:
back.recv()
#timeout events are used for pupil sync.
#annouce masterhood every interval time:
if isinstance(self.time_sync_node, Clock_Sync_Master):
n.shouts(
self.group, SYNC_TIME_MASTER_ANNOUNCE +
"%s" % self.clock_master_worthiness() + msg_delimeter +
'%s' % self.time_sync_node.port)
# synced slave: see if we should become master if we dont hear annoncement within time.
elif isinstance(self.time_sync_node, Clock_Sync_Follower
) and not self.time_sync_node.offset_remains:
if self.get_unadjusted_time(
) - self.last_master_announce > self.time_sync_wait_interval_short:
self.time_sync_node.terminate()
self.time_sync_node = Clock_Sync_Master(
time_fn=self.get_time)
n.shouts(
self.group, SYNC_TIME_MASTER_ANNOUNCE +
"%s" % self.clock_master_worthiness() +
msg_delimeter + '%s' % self.time_sync_node.port)
# unsynced slave or none should wait longer but eventually take over
elif self.get_unadjusted_time(
) - self.last_master_announce > self.time_sync_wait_interval_long:
if self.time_sync_node:
self.time_sync_node.terminate()
self.time_sync_node = Clock_Sync_Master(
time_fn=self.get_time)
n.shouts(
self.group, SYNC_TIME_MASTER_ANNOUNCE +
"%s" % self.clock_master_worthiness() + msg_delimeter +
'%s' % self.time_sync_node.port)
t = Timer(self.time_sync_announce_interval, wake_up)
t.daemon = True
t.start()
if pipe in items and items[pipe] == zmq.POLLIN:
message = pipe.recv()
# message to quit
if message.decode('utf-8') == exit_thread:
break
else:
logger.debug("Shout '%s' to '%s' " % (message, self.group))
n.shouts(self.group, message)
if n.socket() in items and items[n.socket()] == zmq.POLLIN:
cmds = n.recv()
msg_type = cmds.pop(0)
msg_type = msg_type.decode('utf-8')
if msg_type == "SHOUT":
uuid, name, group, msg = cmds
logger.debug("'%s' shouts '%s'." % (name, msg))
self._handle_msg(uuid, name, msg, n)
elif msg_type == "WHISPER":
uuid, name, msg = cmds
logger.debug("'%s/' whispers '%s'." % (name, msg))
self._handle_msg_whisper(uuid, name, msg, n)
elif msg_type == "JOIN":
uuid, name, group = cmds
if group == self.group:
self.group_members[uuid] = name
self.update_gui()
elif msg_type == "EXIT":
uuid, name = cmds
try:
del self.group_members[uuid]
except KeyError:
pass
else:
self.update_gui()
# elif msg_type == "LEAVE":
# uuid,name,group = cmds
# elif msg_type == "ENTER":
# uuid,name,headers,ip = cmds
# logger.warning((uuid,'name',headers,ip))
else:
pass
logger.debug('thread_loop closing.')
self.thread_pipe = None
n.stop()
class Time_Sync(Plugin):
"""Synchronize time of Actors
across local network.
"""
def __init__(self, g_pool):
super(Time_Sync, self).__init__(g_pool)
self.menu = None
#variables for the time sync logic
self.time_sync_node = None
#constants for the time sync logic
self._ti_break = random.random()/10.
self.master_announce_interval = 5
self.master_announce_timeout = self.master_announce_interval * 4
self.master_announce_timeout_notification = {'subject':"time_sync.master_announce_timeout", 'delay':self.master_announce_timeout}
self.master_announce_interval_notification = {'subject':"time_sync.master_announce_interval", 'delay':self.master_announce_interval}
self.notify_all(self.master_announce_timeout_notification)
@property
def is_master(self):
return isinstance(self.time_sync_node,Clock_Sync_Master)
@property
def is_follower(self):
return isinstance(self.time_sync_node,Clock_Sync_Follower)
@property
def is_nothing(self):
return self.time_sync_node is None
def clock_master_worthiness(self):
'''
How worthy am I to be the clock master?
A measure 0 (unworthy) to 1 (destined)
range is from 0. - 0.9 the rest is reserved for ti-breaking
'''
worthiness = 0.
if self.g_pool.timebase.value != 0:
worthiness += 0.4
worthiness +=self._ti_break
return worthiness
###time sync fns these are used by the time sync node to get and adjust time
def get_unadjusted_time(self):
#return time not influced by outside clocks.
return self.g_pool.get_now()
def get_time(self):
return self.g_pool.get_timestamp()
def slew_time(self,offset):
self.g_pool.timebase.value +=offset
def jump_time(self,offset):
ok_to_change = True
for p in self.g_pool.plugins:
if p.class_name == 'Recorder':
if p.running:
ok_to_change = False
logger.error("Request to change timebase during recording ignored. Turn off recording first.")
break
if ok_to_change:
self.slew_time(offset)
logger.info("Pupil Sync has adjusted the clock by %ss"%offset)
return True
else:
return False
def init_gui(self):
def close():
self.alive = False
def sync_status_info():
if self.time_sync_node is None:
return 'Waiting for time sync msg.'
else:
return str(self.time_sync_node)
help_str = "Synchonize time of Pupil captures across the local network."
self.menu = ui.Growing_Menu('Network Time Sync')
self.menu.append(ui.Button('Close',close))
self.menu.append(ui.Info_Text(help_str))
help_str = "All pupil nodes of one group share a Master clock."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(ui.Text_Input('sync status',getter=sync_status_info,setter=lambda _: _))
# self.menu[-1].read_only = True
self.g_pool.sidebar.append(self.menu)
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
def on_notify(self,notification):
"""Synchronize time of Actors across local network.
The notification scheme is used to handle interal timing
and to talk to remote pers via the `Pupil_Groups` plugin.
Reacts to notifications:
``time_sync.master_announcement``: React accordingly to annouce notification from remote peer.
``time_sync.master_announce_interval``: Re-annouce clock masterhood.
``time_sync.master_announce_timeout``: React accordingly when no master announcement has appeard whithin timeout.
Emits notifications:
``time_sync.master_announcement``: Announce masterhood to remote peers (remote notification).
``time_sync.master_announce_interval``: Re-announce masterhood reminder (delayed notification).
``time_sync.master_announce_timeout``: Timeout for foreind master announcement (delayed notification).
"""
if notification['subject'].startswith('time_sync.master_announcement'):
if self.is_master:
if notification['worthiness'] > self.clock_master_worthiness():
#We need to yield.
self.time_sync_node.stop()
self.time_sync_node = None
else:
#Denounce the lesser competition.
n = { 'subject':'time_sync.master_announcement',
'host':self.time_sync_node.host,
'port':self.time_sync_node.port,
'worthiness':self.clock_master_worthiness(),
'remote_notify':'all'
}
self.notify_all(n)
if self.is_follower:
# update follower info
self.time_sync_node.host = notification['host']
self.time_sync_node.port = notification['port']
if self.is_nothing:
# Create follower.
logger.debug("Clock will sync with %s"%notification['host'])
self.time_sync_node = Clock_Sync_Follower(notification['host'],port=notification['port'],interval=10,time_fn=self.get_time,jump_fn=self.jump_time,slew_fn=self.slew_time)
if not self.is_master:
#(Re)set the timer.
self.notify_all(self.master_announce_timeout_notification)
elif notification['subject'].startswith('time_sync.master_announce_timeout'):
if self.is_master:
pass
else:
#We have not heard from a master in too long.
logger.info("Elevate self to clock master.")
self.time_sync_node = Clock_Sync_Master(self.g_pool.get_timestamp)
n = { 'subject':'time_sync.master_announcement',
'host':self.time_sync_node.host,
'port':self.time_sync_node.port,
'worthiness':self.clock_master_worthiness(),
'remote_notify':'all'
}
self.notify_all(n)
self.notify_all(self.master_announce_interval_notification)
elif notification['subject'].startswith('time_sync.master_announce_interval'):
# The time has come to remind others of our master hood.
if self.is_master:
n = { 'subject':'time_sync.master_announcement',
'host':self.time_sync_node.host,
'port':self.time_sync_node.port,
'worthiness':self.clock_master_worthiness(),
'remote_notify':'all' }
self.notify_all(n)
# Set the next annouce timer.
self.notify_all(self.master_announce_interval_notification)
def get_init_dict(self):
return {}
def cleanup(self):
if self.time_sync_node:
self.time_sync_node.terminate()
self.deinit_gui()
def on_notify(self,notification):
"""Synchronize time of Actors across local network.
The notification scheme is used to handle interal timing
and to talk to remote pers via the `Pupil_Groups` plugin.
Reacts to notifications:
``time_sync.master_announcement``: React accordingly to annouce notification from remote peer.
``time_sync.master_announce_interval``: Re-annouce clock masterhood.
``time_sync.master_announce_timeout``: React accordingly when no master announcement has appeard whithin timeout.
Emits notifications:
``time_sync.master_announcement``: Announce masterhood to remote peers (remote notification).
``time_sync.master_announce_interval``: Re-announce masterhood reminder (delayed notification).
``time_sync.master_announce_timeout``: Timeout for foreind master announcement (delayed notification).
"""
if notification['subject'].startswith('time_sync.master_announcement'):
if self.is_master:
if notification['worthiness'] > self.clock_master_worthiness():
#We need to yield.
self.time_sync_node.stop()
self.time_sync_node = None
else:
#Denounce the lesser competition.
n = { 'subject':'time_sync.master_announcement',
'host':self.time_sync_node.host,
'port':self.time_sync_node.port,
'worthiness':self.clock_master_worthiness(),
'remote_notify':'all'
}
self.notify_all(n)
if self.is_follower:
# update follower info
self.time_sync_node.host = notification['host']
self.time_sync_node.port = notification['port']
if self.is_nothing:
# Create follower.
logger.debug("Clock will sync with %s"%notification['host'])
self.time_sync_node = Clock_Sync_Follower(notification['host'],port=notification['port'],interval=10,time_fn=self.get_time,jump_fn=self.jump_time,slew_fn=self.slew_time)
if not self.is_master:
#(Re)set the timer.
self.notify_all(self.master_announce_timeout_notification)
elif notification['subject'].startswith('time_sync.master_announce_timeout'):
if self.is_master:
pass
else:
#We have not heard from a master in too long.
logger.info("Elevate self to clock master.")
self.time_sync_node = Clock_Sync_Master(self.g_pool.get_timestamp)
n = { 'subject':'time_sync.master_announcement',
'host':self.time_sync_node.host,
'port':self.time_sync_node.port,
'worthiness':self.clock_master_worthiness(),
'remote_notify':'all'
}
self.notify_all(n)
self.notify_all(self.master_announce_interval_notification)
elif notification['subject'].startswith('time_sync.master_announce_interval'):
# The time has come to remind others of our master hood.
if self.is_master:
n = { 'subject':'time_sync.master_announcement',
'host':self.time_sync_node.host,
'port':self.time_sync_node.port,
'worthiness':self.clock_master_worthiness(),
'remote_notify':'all' }
self.notify_all(n)
# Set the next annouce timer.
self.notify_all(self.master_announce_interval_notification)
class Time_Sync(Plugin):
"""Synchronize time of Actors
across local network.
"""
def __init__(self, g_pool):
super().__init__(g_pool)
self.menu = None
#variables for the time sync logic
self.time_sync_node = None
#constants for the time sync logic
self._ti_break = random.random() / 10.
self.master_announce_interval = 5
self.master_announce_timeout = self.master_announce_interval * 4
self.master_announce_timeout_notification = {
'subject': "time_sync.master_announce_timeout",
'delay': self.master_announce_timeout
}
self.master_announce_interval_notification = {
'subject': "time_sync.master_announce_interval",
'delay': self.master_announce_interval
}
self.notify_all(self.master_announce_timeout_notification)
@property
def is_master(self):
return isinstance(self.time_sync_node, Clock_Sync_Master)
@property
def is_follower(self):
return isinstance(self.time_sync_node, Clock_Sync_Follower)
@property
def is_nothing(self):
return self.time_sync_node is None
def clock_master_worthiness(self):
'''
How worthy am I to be the clock master?
A measure 0 (unworthy) to 1 (destined)
range is from 0. - 0.9 the rest is reserved for ti-breaking
'''
worthiness = 0.
if self.g_pool.timebase.value != 0:
worthiness += 0.4
worthiness += self._ti_break
return worthiness
###time sync fns these are used by the time sync node to get and adjust time
def get_unadjusted_time(self):
#return time not influced by outside clocks.
return self.g_pool.get_now()
def get_time(self):
return self.g_pool.get_timestamp()
def slew_time(self, offset):
self.g_pool.timebase.value += offset
def jump_time(self, offset):
ok_to_change = True
for p in self.g_pool.plugins:
if p.class_name == 'Recorder':
if p.running:
ok_to_change = False
logger.error(
"Request to change timebase during recording ignored. Turn off recording first."
)
break
if ok_to_change:
self.slew_time(offset)
logger.info(
"Pupil Sync has adjusted the clock by {}s".format(offset))
return True
else:
return False
def init_gui(self):
def close():
self.alive = False
def sync_status_info():
if self.time_sync_node is None:
return 'Waiting for time sync msg.'
else:
return str(self.time_sync_node)
help_str = "Synchonize time of Pupil captures across the local network."
self.menu = ui.Growing_Menu('Network Time Sync')
self.menu.append(ui.Button('Close', close))
self.menu.append(ui.Info_Text(help_str))
help_str = "All pupil nodes of one group share a Master clock."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(
ui.Text_Input('sync status',
getter=sync_status_info,
setter=lambda _: _))
# self.menu[-1].read_only = True
self.g_pool.sidebar.append(self.menu)
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
def on_notify(self, notification):
"""Synchronize time of Actors across local network.
The notification scheme is used to handle interal timing
and to talk to remote pers via the `Pupil_Groups` plugin.
Reacts to notifications:
``time_sync.master_announcement``: React accordingly to annouce notification from remote peer.
``time_sync.master_announce_interval``: Re-annouce clock masterhood.
``time_sync.master_announce_timeout``: React accordingly when no master announcement has appeard whithin timeout.
Emits notifications:
``time_sync.master_announcement``: Announce masterhood to remote peers (remote notification).
``time_sync.master_announce_interval``: Re-announce masterhood reminder (delayed notification).
``time_sync.master_announce_timeout``: Timeout for foreind master announcement (delayed notification).
"""
if notification['subject'].startswith('time_sync.master_announcement'):
if self.is_master:
if notification['worthiness'] > self.clock_master_worthiness():
#We need to yield.
self.time_sync_node.stop()
self.time_sync_node = None
else:
#Denounce the lesser competition.
n = {
'subject': 'time_sync.master_announcement',
'host': self.time_sync_node.host,
'port': self.time_sync_node.port,
'worthiness': self.clock_master_worthiness(),
'remote_notify': 'all'
}
self.notify_all(n)
if self.is_follower:
# update follower info
self.time_sync_node.host = notification['host']
self.time_sync_node.port = notification['port']
if self.is_nothing:
# Create follower.
logger.debug("Clock will sync with {}".format(
notification['host']))
self.time_sync_node = Clock_Sync_Follower(
notification['host'],
port=notification['port'],
interval=10,
time_fn=self.get_time,
jump_fn=self.jump_time,
slew_fn=self.slew_time)
if not self.is_master:
#(Re)set the timer.
self.notify_all(self.master_announce_timeout_notification)
elif notification['subject'].startswith(
'time_sync.master_announce_timeout'):
if self.is_master:
pass
else:
#We have not heard from a master in too long.
logger.info("Elevate self to clock master.")
self.time_sync_node = Clock_Sync_Master(
self.g_pool.get_timestamp)
n = {
'subject': 'time_sync.master_announcement',
'host': self.time_sync_node.host,
'port': self.time_sync_node.port,
'worthiness': self.clock_master_worthiness(),
'remote_notify': 'all'
}
self.notify_all(n)
self.notify_all(self.master_announce_interval_notification)
elif notification['subject'].startswith(
'time_sync.master_announce_interval'):
# The time has come to remind others of our master hood.
if self.is_master:
n = {
'subject': 'time_sync.master_announcement',
'host': self.time_sync_node.host,
'port': self.time_sync_node.port,
'worthiness': self.clock_master_worthiness(),
'remote_notify': 'all'
}
self.notify_all(n)
# Set the next annouce timer.
self.notify_all(self.master_announce_interval_notification)
def get_init_dict(self):
return {}
def cleanup(self):
if self.time_sync_node:
self.time_sync_node.terminate()
self.deinit_gui()
def on_notify(self, notification):
"""Synchronize time of Actors across local network.
The notification scheme is used to handle interal timing
and to talk to remote pers via the `Pupil_Groups` plugin.
Reacts to notifications:
``time_sync.master_announcement``: React accordingly to annouce notification from remote peer.
``time_sync.master_announce_interval``: Re-annouce clock masterhood.
``time_sync.master_announce_timeout``: React accordingly when no master announcement has appeard whithin timeout.
Emits notifications:
``time_sync.master_announcement``: Announce masterhood to remote peers (remote notification).
``time_sync.master_announce_interval``: Re-announce masterhood reminder (delayed notification).
``time_sync.master_announce_timeout``: Timeout for foreind master announcement (delayed notification).
"""
if notification['subject'].startswith('time_sync.master_announcement'):
if self.is_master:
if notification['worthiness'] > self.clock_master_worthiness():
#We need to yield.
self.time_sync_node.stop()
self.time_sync_node = None
else:
#Denounce the lesser competition.
n = {
'subject': 'time_sync.master_announcement',
'host': self.time_sync_node.host,
'port': self.time_sync_node.port,
'worthiness': self.clock_master_worthiness(),
'remote_notify': 'all'
}
self.notify_all(n)
if self.is_follower:
# update follower info
self.time_sync_node.host = notification['host']
self.time_sync_node.port = notification['port']
if self.is_nothing:
# Create follower.
logger.debug("Clock will sync with {}".format(
notification['host']))
self.time_sync_node = Clock_Sync_Follower(
notification['host'],
port=notification['port'],
interval=10,
time_fn=self.get_time,
jump_fn=self.jump_time,
slew_fn=self.slew_time)
if not self.is_master:
#(Re)set the timer.
self.notify_all(self.master_announce_timeout_notification)
elif notification['subject'].startswith(
'time_sync.master_announce_timeout'):
if self.is_master:
pass
else:
#We have not heard from a master in too long.
logger.info("Elevate self to clock master.")
self.time_sync_node = Clock_Sync_Master(
self.g_pool.get_timestamp)
n = {
'subject': 'time_sync.master_announcement',
'host': self.time_sync_node.host,
'port': self.time_sync_node.port,
'worthiness': self.clock_master_worthiness(),
'remote_notify': 'all'
}
self.notify_all(n)
self.notify_all(self.master_announce_interval_notification)
elif notification['subject'].startswith(
'time_sync.master_announce_interval'):
# The time has come to remind others of our master hood.
if self.is_master:
n = {
'subject': 'time_sync.master_announcement',
'host': self.time_sync_node.host,
'port': self.time_sync_node.port,
'worthiness': self.clock_master_worthiness(),
'remote_notify': 'all'
}
self.notify_all(n)
# Set the next annouce timer.
self.notify_all(self.master_announce_interval_notification)
class Time_Sync(Plugin):
"""Synchronize time across local network.
Implements the Pupil Time Sync protocol.
Acts as clock service and as follower if required.
See `time_sync_spec.md` for details.
"""
icon_chr = chr(0xec15)
icon_font = 'pupil_icons'
def __init__(self, g_pool, node_name=None, sync_group_prefix='default', base_bias=1.):
super().__init__(g_pool)
self.sync_group_prefix = sync_group_prefix
self.discovery = None
self.leaderboard = []
self.has_been_master = 0.
self.has_been_synced = 0.
self.tie_breaker = random.random()
self.base_bias = base_bias
self.master_service = Clock_Sync_Master(self.g_pool.get_timestamp)
self.follower_service = None # only set if there is a better server than us
self.restart_discovery(node_name)
@property
def sync_group(self):
return self.sync_group_prefix + '-time_sync-' + __protocol_version__
@sync_group.setter
def sync_group(self, full_name):
self.sync_group_prefix = full_name.rsplit('-time_sync-' + __protocol_version__, maxsplit=1)[0]
def init_ui(self):
self.add_menu()
self.menu.label = 'Network Time Sync'
help_str = "Synchonize time of Pupil Captures across the local network."
self.menu.append(ui.Info_Text('Protocol version: ' + __protocol_version__))
self.menu.append(ui.Info_Text(help_str))
help_str = "All pupil nodes of one group share a Master clock."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(ui.Text_Input('node_name', self, label='Node Name', setter=self.restart_discovery))
self.menu.append(ui.Text_Input('sync_group_prefix', self, label='Sync Group', setter=self.change_sync_group))
def sync_status():
if self.follower_service:
return str(self.follower_service)
else:
return 'Clock Master'
self.menu.append(ui.Text_Input('sync status', getter=sync_status, setter=lambda _: _, label='Status'))
def set_bias(bias):
if bias < 0:
bias = 0.
self.base_bias = bias
self.announce_clock_master_info()
self.evaluate_leaderboard()
help_str = "The clock service with the highest bias becomes clock master."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(ui.Text_Input('base_bias', self, label='Master Bias', setter=set_bias))
self.menu.append(ui.Text_Input('leaderboard', self, label='Master Nodes in Group'))
def recent_events(self, events):
should_announce = False
for evt in self.discovery.recent_events():
if evt.type == 'SHOUT':
try:
self.update_leaderboard(evt.peer_uuid, evt.peer_name, float(evt.msg[0]), int(evt.msg[1]))
except Exception as e:
logger.debug('Garbage raised `{}` -- dropping.'.format(e))
self.evaluate_leaderboard()
elif evt.type == 'JOIN' and evt.group == self.sync_group:
should_announce = True
elif (evt.type == 'LEAVE' and evt.group == self.sync_group) or evt.type == 'EXIT':
self.remove_from_leaderboard(evt.peer_uuid)
self.evaluate_leaderboard()
if should_announce:
self.announce_clock_master_info()
if not self.has_been_synced and self.follower_service and self.follower_service.in_sync:
self.has_been_synced = 1.
self.announce_clock_master_info()
self.evaluate_leaderboard()
def update_leaderboard(self, uuid, name, rank, port):
for cs in self.leaderboard:
if cs.uuid == uuid:
if (cs.rank != rank) or (cs.port != port):
self.remove_from_leaderboard(cs.uuid)
break
else:
# no changes. Just leave as is
return
# clock service was not encountered before or has changed adding it to leaderboard
cs = Clock_Service(uuid, name, rank, port)
heappush(self.leaderboard, cs)
logger.debug('{} added'.format(cs))
def remove_from_leaderboard(self, uuid):
for cs in self.leaderboard:
if cs.uuid == uuid:
self.leaderboard.remove(cs)
logger.debug('{} removed'.format(cs))
break
def evaluate_leaderboard(self):
if not self.leaderboard:
logger.debug("nobody on the leader board.")
return
current_leader = self.leaderboard[0]
if self.discovery.uuid() != current_leader.uuid:
# we are not the leader!
leader_ep = self.discovery.peer_address(current_leader.uuid)
leader_addr = urlparse(leader_ep).netloc.split(':')[0]
if self.follower_service is None:
# make new follower
self.follower_service = Clock_Sync_Follower(leader_addr,
port=current_leader.port,
interval=10,
time_fn=self.get_time,
jump_fn=self.jump_time,
slew_fn=self.slew_time)
else:
# update follower_service
self.follower_service.host = leader_addr
self.follower_service.port = current_leader.port
return
# we are the leader
logger.debug("we are the leader")
if self.follower_service is not None:
self.follower_service.terminate()
self.follower_service = None
if not self.has_been_master:
self.has_been_master = 1.
logger.debug('Become clock master with rank {}'.format(self.rank))
self.announce_clock_master_info()
def announce_clock_master_info(self):
self.discovery.shout(self.sync_group, [repr(self.rank).encode(),
repr(self.master_service.port).encode()])
self.update_leaderboard(self.discovery.uuid(), self.node_name, self.rank, self.master_service.port)
@property
def rank(self):
return 4*self.base_bias + 2*self.has_been_master + self.has_been_synced + self.tie_breaker
def get_time(self):
return self.g_pool.get_timestamp()
def slew_time(self, offset):
self.g_pool.timebase.value += offset
def jump_time(self, offset):
ok_to_change = True
for p in self.g_pool.plugins:
if p.class_name == 'Recorder':
if p.running:
ok_to_change = False
logger.error("Request to change timebase during recording ignored. Turn off recording first.")
break
if ok_to_change:
self.slew_time(offset)
logger.info("Pupil Sync has adjusted the clock by {}s".format(offset))
return True
else:
return False
def restart_discovery(self, name):
if self.discovery:
if self.discovery.name() == name:
return
else:
self.discovery.leave(self.sync_group)
self.discovery.stop()
self.leaderboard = []
self.node_name = name or gethostname()
self.discovery = Pyre(self.node_name)
# Either joining network for the first time or rejoining the same group.
self.discovery.join(self.sync_group)
self.discovery.start()
self.announce_clock_master_info()
def change_sync_group(self, new_group_prefix):
if new_group_prefix != self.sync_group_prefix:
self.discovery.leave(self.sync_group)
self.leaderboard = []
if self.follower_service:
self.follower_service.terminate()
self.follower = None
self.sync_group_prefix = new_group_prefix
self.discovery.join(self.sync_group)
self.announce_clock_master_info()
def deinit_ui(self):
self.remove_menu()
def get_init_dict(self):
return {'node_name': self.node_name,
'sync_group_prefix': self.sync_group_prefix,
'base_bias': self.base_bias}
def cleanup(self):
self.discovery.leave(self.sync_group)
self.discovery.stop()
self.master_service.stop()
if self.follower_service:
self.follower_service.stop()
self.master_service = None
self.follower_service = None
self.discovery = None
class Time_Sync(Plugin):
"""Synchronize time across local network.
Implements the Pupil Time Sync protocol.
Acts as clock service and as follower if required.
See `time_sync_spec.md` for details.
"""
def __init__(self, g_pool, node_name=None, sync_group_prefix='default', base_bias=1.):
super().__init__(g_pool)
self.menu = None
self.sync_group_prefix = sync_group_prefix
self.discovery = None
self.leaderboard = []
self.has_been_master = 0.
self.has_been_synced = 0.
self.tie_breaker = random.random()
self.base_bias = base_bias
self.master_service = Clock_Sync_Master(self.g_pool.get_timestamp)
self.follower_service = None # only set if there is a better server than us
self.restart_discovery(node_name)
@property
def sync_group(self):
return self.sync_group_prefix + '-time_sync-' + __protocol_version__
@sync_group.setter
def sync_group(self, full_name):
self.sync_group_prefix = full_name.rsplit('-time_sync-' + __protocol_version__, maxsplit=1)[0]
def init_gui(self):
def close():
self.alive = False
help_str = "Synchonize time of Pupil Captures across the local network."
self.menu = ui.Growing_Menu('Network Time Sync')
self.menu.collapsed = True
self.menu.append(ui.Button('Close', close))
self.menu.append(ui.Info_Text('Protocol version: ' + __protocol_version__))
self.menu.append(ui.Info_Text(help_str))
help_str = "All pupil nodes of one group share a Master clock."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(ui.Text_Input('node_name', self, label='Node Name', setter=self.restart_discovery))
self.menu.append(ui.Text_Input('sync_group_prefix', self, label='Sync Group', setter=self.change_sync_group))
def sync_status():
if self.follower_service:
return str(self.follower_service)
else:
return 'Clock Master'
self.menu.append(ui.Text_Input('sync status', getter=sync_status, setter=lambda _: _, label='Status'))
def set_bias(bias):
if bias < 0:
bias = 0.
self.base_bias = bias
self.announce_clock_master_info()
self.evaluate_leaderboard()
help_str = "The clock service with the highest bias becomes clock master."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(ui.Text_Input('base_bias', self, label='Master Bias', setter=set_bias))
self.menu.append(ui.Text_Input('leaderboard', self, label='Master Nodes in Group'))
self.g_pool.sidebar.append(self.menu)
def recent_events(self, events):
should_announce = False
for evt in self.discovery.recent_events():
if evt.type == 'SHOUT':
try:
self.update_leaderboard(evt.peer_uuid, evt.peer_name, float(evt.msg[0]), int(evt.msg[1]))
except Exception as e:
logger.debug('Garbage raised `{}` -- dropping.'.format(e))
self.evaluate_leaderboard()
elif evt.type == 'JOIN' and evt.group == self.sync_group:
should_announce = True
elif (evt.type == 'LEAVE' and evt.group == self.sync_group) or evt.type == 'EXIT':
self.remove_from_leaderboard(evt.peer_uuid)
self.evaluate_leaderboard()
if should_announce:
self.announce_clock_master_info()
if not self.has_been_synced and self.follower_service and self.follower_service.in_sync:
self.has_been_synced = 1.
self.announce_clock_master_info()
self.evaluate_leaderboard()
def update_leaderboard(self, uuid, name, rank, port):
for cs in self.leaderboard:
if cs.uuid == uuid:
if (cs.rank != rank) or (cs.port != port):
self.remove_from_leaderboard(cs.uuid)
break
else:
# no changes. Just leave as is
return
# clock service was not encountered before or has changed adding it to leaderboard
cs = Clock_Service(uuid, name, rank, port)
heappush(self.leaderboard, cs)
logger.debug('{} added'.format(cs))
def remove_from_leaderboard(self, uuid):
for cs in self.leaderboard:
if cs.uuid == uuid:
self.leaderboard.remove(cs)
logger.debug('{} removed'.format(cs))
break
def evaluate_leaderboard(self):
if not self.leaderboard:
logger.debug("nobody on the leader board.")
return
current_leader = self.leaderboard[0]
if self.discovery.uuid() != current_leader.uuid:
# we are not the leader!
leader_ep = self.discovery.peer_address(current_leader.uuid)
leader_addr = urlparse(leader_ep).netloc.split(':')[0]
if self.follower_service is None:
# make new follower
self.follower_service = Clock_Sync_Follower(leader_addr,
port=current_leader.port,
interval=10,
time_fn=self.get_time,
jump_fn=self.jump_time,
slew_fn=self.slew_time)
else:
# update follower_service
self.follower_service.host = leader_addr
self.follower_service.port = current_leader.port
return
# we are the leader
logger.debug("we are the leader")
if self.follower_service is not None:
self.follower_service.terminate()
self.follower_service = None
if not self.has_been_master:
self.has_been_master = 1.
logger.debug('Become clock master with rank {}'.format(self.rank))
self.announce_clock_master_info()
def announce_clock_master_info(self):
self.discovery.shout(self.sync_group, [repr(self.rank).encode(),
repr(self.master_service.port).encode()])
self.update_leaderboard(self.discovery.uuid(), self.node_name, self.rank, self.master_service.port)
@property
def rank(self):
return 4*self.base_bias + 2*self.has_been_master + self.has_been_synced + self.tie_breaker
def get_time(self):
return self.g_pool.get_timestamp()
def slew_time(self, offset):
self.g_pool.timebase.value += offset
def jump_time(self, offset):
ok_to_change = True
for p in self.g_pool.plugins:
if p.class_name == 'Recorder':
if p.running:
ok_to_change = False
logger.error("Request to change timebase during recording ignored. Turn off recording first.")
break
if ok_to_change:
self.slew_time(offset)
logger.info("Pupil Sync has adjusted the clock by {}s".format(offset))
return True
else:
return False
def restart_discovery(self, name):
if self.discovery:
if self.discovery.name() == name:
return
else:
self.discovery.leave(self.sync_group)
self.discovery.stop()
self.leaderboard = []
self.node_name = name or gethostname()
self.discovery = Pyre(self.node_name)
# Either joining network for the first time or rejoining the same group.
self.discovery.join(self.sync_group)
self.discovery.start()
self.announce_clock_master_info()
def change_sync_group(self, new_group_prefix):
if new_group_prefix != self.sync_group_prefix:
self.discovery.leave(self.sync_group)
self.leaderboard = []
if self.follower_service:
self.follower_service.terminate()
self.follower = None
self.sync_group_prefix = new_group_prefix
self.discovery.join(self.sync_group)
self.announce_clock_master_info()
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
def get_init_dict(self):
return {'node_name': self.node_name,
'sync_group_prefix': self.sync_group_prefix,
'base_bias': self.base_bias}
def cleanup(self):
self.deinit_gui()
self.discovery.leave(self.sync_group)
self.discovery.stop()
self.master_service.stop()
if self.follower_service:
self.follower_service.stop()
self.master_service = None
self.follower_service = None
self.discovery = None
