def run_time_sync_master(group):
pts_group = group + '-time_sync-v1'
# the time source in the example is python time.time you can change this.
# replace with an implementation that give your custom time in floating sec.
clock_service = Clock_Sync_Master(time)
# This example is a clock service only, not a clock follower.
# Therefore the rank is designed to always trump all others.
rank = 1000
discovery = Pyre('pupil-helper-service')
discovery.join(pts_group)
discovery.start()
logger.info('Joining "{}" group with rank {}'.format(pts_group, rank))
def announce_clock_service_info():
discovery.shout(pts_group, [repr(rank).encode(), repr(clock_service.port).encode()])
try:
for event in discovery.events():
if event.type == 'JOIN' and event.group == pts_group:
logger.info('"{}" joined "{}" group. Announcing service.'.format(event.peer_name, pts_group))
announce_clock_service_info()
except KeyboardInterrupt:
pass
finally:
logger.info('Leaving "{}" group'.format(pts_group))
discovery.leave(pts_group)
discovery.stop()
clock_service.stop()
def run_time_sync_master(group):
pts_group = group + '-time_sync-v1'
# the time source in the example is python time.time you can change this.
# replace with an implementation that give your custom time in floating sec.
clock_service = Clock_Sync_Master(time)
# This example is a clock service only, not a clock follower.
# Therefore the rank is designed to always trump all others.
rank = 1000
discovery = Pyre('pupil-helper-service')
discovery.join(pts_group)
discovery.start()
logger.info('Joining "{}" group with rank {}'.format(pts_group, rank))
def announce_clock_service_info():
discovery.shout(
pts_group,
[repr(rank).encode(),
repr(clock_service.port).encode()])
try:
for event in discovery.events():
if event.type == 'JOIN' and event.group == pts_group:
logger.info(
'"{}" joined "{}" group. Announcing service.'.format(
event.peer_name, pts_group))
announce_clock_service_info()
except KeyboardInterrupt:
pass
finally:
logger.info('Leaving "{}" group'.format(pts_group))
discovery.leave(pts_group)
discovery.stop()
clock_service.stop()
class _NetworkNode(NetworkInterface):
"""
Communication node
Creates Pyre node and handles all communication.
"""
def __init__(self,
format: DataFormat,
context=None,
name=None,
headers=(),
callbacks=()):
self._name = name
self._format = format
self._headers = headers
self._pyre_node = None
self._context = context or zmq.Context()
self._sensors_by_host = {}
self._callbacks = [self._on_event] + list(callbacks)
# Public NetworkInterface API
@property
def has_events(self) -> bool:
return self.running and self._pyre_node.socket().get(
zmq.EVENTS) & zmq.POLLIN
@property
def running(self) -> bool:
return bool(self._pyre_node)
@property
def sensors(self) -> typing.Mapping[str, NetworkSensor]:
sensors = {}
for sensor in self._sensors_by_host.values():
sensors.update(sensor)
return sensors
@property
def callbacks(self) -> typing.Iterable[NetworkEventCallback]:
return self._callbacks
@callbacks.setter
def callbacks(self, value: typing.Iterable[NetworkEventCallback]):
self._callbacks = value
def start(self):
# Setup node
logger.debug("Starting network...")
self._pyre_node = Pyre(self._name)
self._name = self._pyre_node.name()
for header in self._headers:
self._pyre_node.set_header(*header)
self._pyre_node.join(self._group)
self._pyre_node.start()
def whisper(self, peer, msg_p):
if self._format == DataFormat.V3:
return # no-op
elif self._format == DataFormat.V4:
self._pyre_node.whisper(peer, msg_p)
else:
raise NotImplementedError()
def rejoin(self):
for sensor_uuid, sensor in list(self.sensors.items()):
self._execute_callbacks({
"subject": "detach",
"sensor_uuid": sensor_uuid,
"sensor_name": sensor["sensor_name"],
"host_uuid": sensor["host_uuid"],
"host_name": sensor["host_name"],
})
self._pyre_node.leave(self._group)
self._pyre_node.join(self._group)
def stop(self):
logger.debug("Stopping network...")
self._pyre_node.leave(self._group)
self._pyre_node.stop()
self._pyre_node = None
def handle_event(self):
if not self.has_events:
return
event = PyreEvent(self._pyre_node)
uuid = event.peer_uuid
if event.type == "SHOUT" or event.type == "WHISPER":
try:
payload = event.msg.pop(0).decode()
msg = serial.loads(payload)
msg["subject"]
msg["sensor_uuid"]
msg["host_uuid"] = event.peer_uuid.hex
msg["host_name"] = event.peer_name
except serial.decoder.JSONDecodeError:
logger.warning('Malformatted message: "{}"'.format(payload))
except (ValueError, KeyError):
logger.warning("Malformatted message: {}".format(msg))
except Exception:
logger.debug(tb.format_exc())
else:
if msg["subject"] == "attach":
if self.sensors.get(msg["sensor_uuid"]):
# Sensor already attached. Drop event
return
sensor_type = SensorType.supported_sensor_type_from_str(
msg["sensor_type"])
if sensor_type is None:
logger.debug("Unsupported sensor type: {}".format(
msg["sensor_type"]))
return
elif msg["subject"] == "detach":
sensor_entry = self.sensors.get(msg["sensor_uuid"])
# Check if sensor has been detached already
if not sensor_entry:
return
msg.update(sensor_entry)
else:
logger.debug("Unknown host message: {}".format(msg))
return
self._execute_callbacks(msg)
elif event.type == "JOIN":
# possible values for `group_version`
# - [<unrelated group>]
# - [<unrelated group>, <unrelated version>]
# - ['pupil-mobile']
# - ['pupil-mobile', <version>]
group_version = event.group.split("-v")
group = group_version[0]
version = group_version[1] if len(group_version) > 1 else "0"
elif event.type == "EXIT":
gone_peer = event.peer_uuid.hex
for host_uuid, sensors in list(self._sensors_by_host.items()):
if host_uuid != gone_peer:
continue
for sensor_uuid, sensor in list(sensors.items()):
self._execute_callbacks({
"subject":
"detach",
"sensor_uuid":
sensor_uuid,
"sensor_name":
sensor["sensor_name"],
"host_uuid":
host_uuid,
"host_name":
sensor["host_name"],
})
else:
logger.debug("Dropping {}".format(event))
def sensor(
self,
sensor_uuid: str,
callbacks: typing.Iterable[NetworkEventCallback] = ()
) -> Sensor:
try:
sensor_settings = self.sensors[sensor_uuid].copy()
except KeyError:
raise ValueError(
'"{}" is not an available sensor id.'.format(sensor_uuid))
sensor_type_str = sensor_settings.pop("sensor_type", "unknown")
sensor_type = SensorType.supported_sensor_type_from_str(
sensor_type_str)
if sensor_type is None:
raise ValueError('Sensor of type "{}" is not supported.'.format(
sensor_type_str))
return Sensor.create_sensor(
sensor_type=sensor_type,
format=self._format,
context=self._context,
callbacks=callbacks,
**sensor_settings,
)
# Public
def __str__(self):
return "<{} {} [{}]>".format(__name__, self._name,
self._pyre_node.uuid().hex)
# Private
@property
def _group(self) -> str:
return group_name_from_format(self._format)
def _execute_callbacks(self, event):
for callback in self.callbacks:
callback(self, event)
def _on_event(self, caller, event):
if event["subject"] == "attach":
subject_less = event.copy()
del subject_less["subject"]
host_uuid = event["host_uuid"]
host_sensor = {event["sensor_uuid"]: subject_less}
try:
self._sensors_by_host[host_uuid].update(host_sensor)
except KeyError:
self._sensors_by_host[host_uuid] = host_sensor
logger.debug(f'Attached {host_uuid}.{event["sensor_uuid"]}')
elif event["subject"] == "detach":
for host_uuid, sensors in self._sensors_by_host.items():
try:
del sensors[event["sensor_uuid"]]
logger.debug(
f'Detached {host_uuid}.{event["sensor_uuid"]}')
except KeyError:
pass
hosts_to_remove = [
host_uuid
for host_uuid, sensors in self._sensors_by_host.items()
if len(sensors) == 0
]
for host_uuid in hosts_to_remove:
del self._sensors_by_host[host_uuid]
class Time_Sync(Plugin):
"""Synchronize time across local network.
Implements the Pupil Time Sync protocol.
Acts as clock service and as follower if required.
See `time_sync_spec.md` for details.
"""
icon_chr = chr(0xEC15)
icon_font = "pupil_icons"
def __init__(self,
g_pool,
node_name=None,
sync_group_prefix="default",
base_bias=1.0):
super().__init__(g_pool)
self.sync_group_prefix = sync_group_prefix
self.discovery = None
self.leaderboard = []
self.has_been_master = 0.0
self.has_been_synced = 0.0
self.tie_breaker = random.random()
self.base_bias = base_bias
self.sync_group_members = {}
self.master_service = Clock_Sync_Master(self.g_pool.get_timestamp)
self.follower_service = None # only set if there is a better server than us
self.restart_discovery(node_name)
@property
def sync_group(self):
return self.sync_group_prefix + "-time_sync-" + __protocol_version__
@sync_group.setter
def sync_group(self, full_name):
self.sync_group_prefix = full_name.rsplit("-time_sync-" +
__protocol_version__,
maxsplit=1)[0]
def init_ui(self):
self.add_menu()
self.menu.label = "Network Time Sync"
help_str = "Synchonize time of Pupil Captures across the local network."
self.menu.append(
ui.Info_Text("Protocol version: " + __protocol_version__))
self.menu.append(ui.Info_Text(help_str))
help_str = "All pupil nodes of one group share a Master clock."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(
ui.Text_Input("node_name",
self,
label="Node Name",
setter=self.restart_discovery))
self.menu.append(
ui.Text_Input(
"sync_group_prefix",
self,
label="Sync Group",
setter=self.change_sync_group,
))
def sync_status():
if self.follower_service:
return str(self.follower_service)
else:
return "Clock Master"
self.menu.append(
ui.Text_Input("sync status",
getter=sync_status,
setter=lambda _: _,
label="Status"))
def set_bias(bias):
if bias < 0:
bias = 0.0
self.base_bias = bias
self.announce_clock_master_info()
self.evaluate_leaderboard()
help_str = "The clock service with the highest bias becomes clock master."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(
ui.Text_Input("base_bias",
self,
label="Master Bias",
setter=set_bias))
self.menu.append(
ui.Text_Input("leaderboard", self, label="Master Nodes in Group"))
self.sync_group_members_menu = ui.Growing_Menu("Sync Group Members")
self.menu.append(self.sync_group_members_menu)
def recent_events(self, events):
should_announce = False
for evt in self.discovery.recent_events():
if evt.type == "SHOUT":
try:
self.update_leaderboard(evt.peer_uuid, evt.peer_name,
float(evt.msg[0]), int(evt.msg[1]))
except Exception as e:
logger.debug("Garbage raised `{}` -- dropping.".format(e))
self.evaluate_leaderboard()
elif evt.type == "JOIN" and evt.group == self.sync_group:
should_announce = True
self.insert_sync_group_member(evt.peer_uuid, evt.peer_name)
elif (evt.type == "LEAVE"
and evt.group == self.sync_group) or evt.type == "EXIT":
self.remove_from_leaderboard(evt.peer_uuid)
self.evaluate_leaderboard()
self.remove_sync_group_member(evt.peer_uuid)
if should_announce:
self.announce_clock_master_info()
if (not self.has_been_synced and self.follower_service
and self.follower_service.in_sync):
self.has_been_synced = 1.0
self.announce_clock_master_info()
self.evaluate_leaderboard()
def update_leaderboard(self, uuid, name, rank, port):
for cs in self.leaderboard:
if cs.uuid == uuid:
if (cs.rank != rank) or (cs.port != port):
self.remove_from_leaderboard(cs.uuid)
break
else:
# no changes. Just leave as is
return
# clock service was not encountered before or has changed adding it to leaderboard
cs = Clock_Service(uuid, name, rank, port)
heappush(self.leaderboard, cs)
logger.debug("{} added".format(cs))
def remove_from_leaderboard(self, uuid):
for cs in self.leaderboard:
if cs.uuid == uuid:
self.leaderboard.remove(cs)
logger.debug("{} removed".format(cs))
break
def evaluate_leaderboard(self):
if not self.leaderboard:
logger.debug("nobody on the leader board.")
return
current_leader = self.leaderboard[0]
if self.discovery.uuid() != current_leader.uuid:
# we are not the leader!
leader_ep = self.discovery.peer_address(current_leader.uuid)
leader_addr = urlparse(leader_ep).netloc.split(":")[0]
if self.follower_service is None:
# make new follower
self.follower_service = Clock_Sync_Follower(
leader_addr,
port=current_leader.port,
interval=10,
time_fn=self.get_time,
jump_fn=self.jump_time,
slew_fn=self.slew_time,
)
else:
# update follower_service
self.follower_service.host = leader_addr
self.follower_service.port = current_leader.port
return
# we are the leader
logger.debug("we are the leader")
if self.follower_service is not None:
self.follower_service.terminate()
self.follower_service = None
if not self.has_been_master:
self.has_been_master = 1.0
logger.debug("Become clock master with rank {}".format(self.rank))
self.announce_clock_master_info()
def insert_sync_group_member(self, uuid, name):
member_text = ui.Info_Text(name)
self.sync_group_members[uuid] = member_text
self.sync_group_members_menu.append(member_text)
self.sync_group_members_menu.elements.sort(
key=lambda text_field: text_field.text)
def insert_all_sync_group_members_from_group(self, group):
for uuid in self.discovery.peers_by_group(group):
name = self.discovery.get_peer_name(uuid)
self.insert_sync_group_member(uuid, name)
def remove_all_sync_group_members(self):
for uuid in list(self.sync_group_members.keys()):
self.remove_sync_group_member(uuid)
def remove_sync_group_member(self, uuid):
try:
self.sync_group_members_menu.remove(self.sync_group_members[uuid])
del self.sync_group_members[uuid]
except KeyError:
logger.debug("Peer has already been removed from members list.")
def announce_clock_master_info(self):
self.discovery.shout(
self.sync_group,
[
repr(self.rank).encode(),
repr(self.master_service.port).encode()
],
)
self.update_leaderboard(self.discovery.uuid(), self.node_name,
self.rank, self.master_service.port)
@property
def rank(self):
return (4 * self.base_bias + 2 * self.has_been_master +
self.has_been_synced + self.tie_breaker)
def get_time(self):
return self.g_pool.get_timestamp()
def slew_time(self, offset):
self.g_pool.timebase.value += offset
def jump_time(self, offset):
ok_to_change = True
for p in self.g_pool.plugins:
if p.class_name == "Recorder":
if p.running:
ok_to_change = False
logger.error(
"Request to change timebase during recording ignored. Turn off recording first."
)
break
if ok_to_change:
self.slew_time(offset)
logger.info(
"Pupil Sync has adjusted the clock by {}s".format(offset))
return True
else:
return False
def restart_discovery(self, name):
if self.discovery:
if self.discovery.name() == name:
return
else:
self.remove_all_sync_group_members()
self.discovery.leave(self.sync_group)
self.discovery.stop()
self.leaderboard = []
self.node_name = name or gethostname()
self.discovery = Pyre(self.node_name)
# Either joining network for the first time or rejoining the same group.
self.discovery.join(self.sync_group)
self.discovery.start()
self.announce_clock_master_info()
def change_sync_group(self, new_group_prefix):
if new_group_prefix != self.sync_group_prefix:
self.remove_all_sync_group_members()
self.discovery.leave(self.sync_group)
self.leaderboard = []
if self.follower_service:
self.follower_service.terminate()
self.follower = None
self.sync_group_prefix = new_group_prefix
self.discovery.join(self.sync_group)
self.insert_all_sync_group_members_from_group(self.sync_group)
self.announce_clock_master_info()
def deinit_ui(self):
for uuid in list(self.sync_group_members.keys()):
self.remove_sync_group_member(uuid)
self.remove_menu()
def get_init_dict(self):
return {
"node_name": self.node_name,
"sync_group_prefix": self.sync_group_prefix,
"base_bias": self.base_bias,
}
def cleanup(self):
self.discovery.leave(self.sync_group)
self.discovery.stop()
self.master_service.stop()
if self.follower_service:
self.follower_service.stop()
self.master_service = None
self.follower_service = None
self.discovery = None
def run_time_sync_follower(time_fn, jump_fn, slew_fn, group):
"""Main follower logic"""
# Start Pyre node and find clock services in `pts_group`
pts_group = group + '-time_sync-v1'
discovery = Pyre('pupil-helper-follower')
discovery.join(pts_group)
discovery.start()
logger.info('Joining "{}" group'.format(pts_group))
# The leaderboard keeps track of all clock services
# and is used to determine the clock master
leaderboard = []
follower_service = None
def update_leaderboard(uuid, name, rank, port):
"""Add or update an existing clock service on the leaderboard"""
for cs in leaderboard:
if cs.uuid == uuid:
if (cs.rank != rank) or (cs.port != port):
remove_from_leaderboard(cs.uuid)
break
else:
# no changes. Just leave as is
return
# clock service was not encountered before or has changed adding it to leaderboard
cs = Clock_Service(uuid, name, rank, port)
heappush(leaderboard, cs)
logger.debug('<{}> added'.format(cs))
def remove_from_leaderboard(uuid):
"""Remove an existing clock service from the leaderboard"""
for cs in leaderboard:
if cs.uuid == uuid:
leaderboard.remove(cs)
logger.debug('<{}> removed'.format(cs))
break
def evaluate_leaderboard(follower_service):
"""
Starts/changes/stops the time follower service according to
who the current clock master is.
"""
if not leaderboard:
logger.debug("nobody on the leader board.")
if follower_service is not None:
follower_service.terminate()
return None
current_leader = leaderboard[0]
leader_ep = discovery.peer_address(current_leader.uuid)
leader_addr = urlparse(leader_ep).netloc.split(':')[0]
logger.info('Following <{}>'.format(current_leader))
if follower_service is None:
# make new follower
follower_service = Clock_Sync_Follower(leader_addr,
port=current_leader.port,
interval=10,
time_fn=time_fn,
jump_fn=jump_fn,
slew_fn=slew_fn)
else:
# update follower_service
follower_service.host = leader_addr
follower_service.port = current_leader.port
return follower_service
try:
# wait for the next Pyre event
for event in discovery.events():
if event.type == 'SHOUT':
# clock service announcement
# ill-formatted messages will be dropped
try:
update_leaderboard(event.peer_uuid,
event.peer_name,
float(event.msg[0]),
int(event.msg[1]))
except Exception as e:
logger.debug('Garbage raised `{}` -- dropping.'.format(e))
follower_service = evaluate_leaderboard(follower_service)
elif ((event.type == 'LEAVE' and event.group == pts_group)
or event.type == 'EXIT'):
remove_from_leaderboard(event.peer_uuid)
follower_service = evaluate_leaderboard(follower_service)
except KeyboardInterrupt:
pass
finally:
discovery.leave(pts_group)
discovery.stop()
if follower_service is not None:
follower_service.terminate()
class Time_Sync(Plugin):
"""Synchronize time across local network.
Implements the Pupil Time Sync protocol.
Acts as clock service and as follower if required.
See `time_sync_spec.md` for details.
"""
icon_chr = chr(0xec15)
icon_font = 'pupil_icons'
def __init__(self, g_pool, node_name=None, sync_group_prefix='default', base_bias=1.):
super().__init__(g_pool)
self.sync_group_prefix = sync_group_prefix
self.discovery = None
self.leaderboard = []
self.has_been_master = 0.
self.has_been_synced = 0.
self.tie_breaker = random.random()
self.base_bias = base_bias
self.master_service = Clock_Sync_Master(self.g_pool.get_timestamp)
self.follower_service = None # only set if there is a better server than us
self.restart_discovery(node_name)
@property
def sync_group(self):
return self.sync_group_prefix + '-time_sync-' + __protocol_version__
@sync_group.setter
def sync_group(self, full_name):
self.sync_group_prefix = full_name.rsplit('-time_sync-' + __protocol_version__, maxsplit=1)[0]
def init_ui(self):
self.add_menu()
self.menu.label = 'Network Time Sync'
help_str = "Synchonize time of Pupil Captures across the local network."
self.menu.append(ui.Info_Text('Protocol version: ' + __protocol_version__))
self.menu.append(ui.Info_Text(help_str))
help_str = "All pupil nodes of one group share a Master clock."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(ui.Text_Input('node_name', self, label='Node Name', setter=self.restart_discovery))
self.menu.append(ui.Text_Input('sync_group_prefix', self, label='Sync Group', setter=self.change_sync_group))
def sync_status():
if self.follower_service:
return str(self.follower_service)
else:
return 'Clock Master'
self.menu.append(ui.Text_Input('sync status', getter=sync_status, setter=lambda _: _, label='Status'))
def set_bias(bias):
if bias < 0:
bias = 0.
self.base_bias = bias
self.announce_clock_master_info()
self.evaluate_leaderboard()
help_str = "The clock service with the highest bias becomes clock master."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(ui.Text_Input('base_bias', self, label='Master Bias', setter=set_bias))
self.menu.append(ui.Text_Input('leaderboard', self, label='Master Nodes in Group'))
def recent_events(self, events):
should_announce = False
for evt in self.discovery.recent_events():
if evt.type == 'SHOUT':
try:
self.update_leaderboard(evt.peer_uuid, evt.peer_name, float(evt.msg[0]), int(evt.msg[1]))
except Exception as e:
logger.debug('Garbage raised `{}` -- dropping.'.format(e))
self.evaluate_leaderboard()
elif evt.type == 'JOIN' and evt.group == self.sync_group:
should_announce = True
elif (evt.type == 'LEAVE' and evt.group == self.sync_group) or evt.type == 'EXIT':
self.remove_from_leaderboard(evt.peer_uuid)
self.evaluate_leaderboard()
if should_announce:
self.announce_clock_master_info()
if not self.has_been_synced and self.follower_service and self.follower_service.in_sync:
self.has_been_synced = 1.
self.announce_clock_master_info()
self.evaluate_leaderboard()
def update_leaderboard(self, uuid, name, rank, port):
for cs in self.leaderboard:
if cs.uuid == uuid:
if (cs.rank != rank) or (cs.port != port):
self.remove_from_leaderboard(cs.uuid)
break
else:
# no changes. Just leave as is
return
# clock service was not encountered before or has changed adding it to leaderboard
cs = Clock_Service(uuid, name, rank, port)
heappush(self.leaderboard, cs)
logger.debug('{} added'.format(cs))
def remove_from_leaderboard(self, uuid):
for cs in self.leaderboard:
if cs.uuid == uuid:
self.leaderboard.remove(cs)
logger.debug('{} removed'.format(cs))
break
def evaluate_leaderboard(self):
if not self.leaderboard:
logger.debug("nobody on the leader board.")
return
current_leader = self.leaderboard[0]
if self.discovery.uuid() != current_leader.uuid:
# we are not the leader!
leader_ep = self.discovery.peer_address(current_leader.uuid)
leader_addr = urlparse(leader_ep).netloc.split(':')[0]
if self.follower_service is None:
# make new follower
self.follower_service = Clock_Sync_Follower(leader_addr,
port=current_leader.port,
interval=10,
time_fn=self.get_time,
jump_fn=self.jump_time,
slew_fn=self.slew_time)
else:
# update follower_service
self.follower_service.host = leader_addr
self.follower_service.port = current_leader.port
return
# we are the leader
logger.debug("we are the leader")
if self.follower_service is not None:
self.follower_service.terminate()
self.follower_service = None
if not self.has_been_master:
self.has_been_master = 1.
logger.debug('Become clock master with rank {}'.format(self.rank))
self.announce_clock_master_info()
def announce_clock_master_info(self):
self.discovery.shout(self.sync_group, [repr(self.rank).encode(),
repr(self.master_service.port).encode()])
self.update_leaderboard(self.discovery.uuid(), self.node_name, self.rank, self.master_service.port)
@property
def rank(self):
return 4*self.base_bias + 2*self.has_been_master + self.has_been_synced + self.tie_breaker
def get_time(self):
return self.g_pool.get_timestamp()
def slew_time(self, offset):
self.g_pool.timebase.value += offset
def jump_time(self, offset):
ok_to_change = True
for p in self.g_pool.plugins:
if p.class_name == 'Recorder':
if p.running:
ok_to_change = False
logger.error("Request to change timebase during recording ignored. Turn off recording first.")
break
if ok_to_change:
self.slew_time(offset)
logger.info("Pupil Sync has adjusted the clock by {}s".format(offset))
return True
else:
return False
def restart_discovery(self, name):
if self.discovery:
if self.discovery.name() == name:
return
else:
self.discovery.leave(self.sync_group)
self.discovery.stop()
self.leaderboard = []
self.node_name = name or gethostname()
self.discovery = Pyre(self.node_name)
# Either joining network for the first time or rejoining the same group.
self.discovery.join(self.sync_group)
self.discovery.start()
self.announce_clock_master_info()
def change_sync_group(self, new_group_prefix):
if new_group_prefix != self.sync_group_prefix:
self.discovery.leave(self.sync_group)
self.leaderboard = []
if self.follower_service:
self.follower_service.terminate()
self.follower = None
self.sync_group_prefix = new_group_prefix
self.discovery.join(self.sync_group)
self.announce_clock_master_info()
def deinit_ui(self):
self.remove_menu()
def get_init_dict(self):
return {'node_name': self.node_name,
'sync_group_prefix': self.sync_group_prefix,
'base_bias': self.base_bias}
def cleanup(self):
self.discovery.leave(self.sync_group)
self.discovery.stop()
self.master_service.stop()
if self.follower_service:
self.follower_service.stop()
self.master_service = None
self.follower_service = None
self.discovery = None
class Time_Sync(Plugin):
"""Synchronize time across local network.
Implements the Pupil Time Sync protocol.
Acts as clock service and as follower if required.
See `time_sync_spec.md` for details.
"""
def __init__(self, g_pool, node_name=None, sync_group_prefix='default', base_bias=1.):
super().__init__(g_pool)
self.menu = None
self.sync_group_prefix = sync_group_prefix
self.discovery = None
self.leaderboard = []
self.has_been_master = 0.
self.has_been_synced = 0.
self.tie_breaker = random.random()
self.base_bias = base_bias
self.master_service = Clock_Sync_Master(self.g_pool.get_timestamp)
self.follower_service = None # only set if there is a better server than us
self.restart_discovery(node_name)
@property
def sync_group(self):
return self.sync_group_prefix + '-time_sync-' + __protocol_version__
@sync_group.setter
def sync_group(self, full_name):
self.sync_group_prefix = full_name.rsplit('-time_sync-' + __protocol_version__, maxsplit=1)[0]
def init_gui(self):
def close():
self.alive = False
help_str = "Synchonize time of Pupil Captures across the local network."
self.menu = ui.Growing_Menu('Network Time Sync')
self.menu.collapsed = True
self.menu.append(ui.Button('Close', close))
self.menu.append(ui.Info_Text('Protocol version: ' + __protocol_version__))
self.menu.append(ui.Info_Text(help_str))
help_str = "All pupil nodes of one group share a Master clock."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(ui.Text_Input('node_name', self, label='Node Name', setter=self.restart_discovery))
self.menu.append(ui.Text_Input('sync_group_prefix', self, label='Sync Group', setter=self.change_sync_group))
def sync_status():
if self.follower_service:
return str(self.follower_service)
else:
return 'Clock Master'
self.menu.append(ui.Text_Input('sync status', getter=sync_status, setter=lambda _: _, label='Status'))
def set_bias(bias):
if bias < 0:
bias = 0.
self.base_bias = bias
self.announce_clock_master_info()
self.evaluate_leaderboard()
help_str = "The clock service with the highest bias becomes clock master."
self.menu.append(ui.Info_Text(help_str))
self.menu.append(ui.Text_Input('base_bias', self, label='Master Bias', setter=set_bias))
self.menu.append(ui.Text_Input('leaderboard', self, label='Master Nodes in Group'))
self.g_pool.sidebar.append(self.menu)
def recent_events(self, events):
should_announce = False
for evt in self.discovery.recent_events():
if evt.type == 'SHOUT':
try:
self.update_leaderboard(evt.peer_uuid, evt.peer_name, float(evt.msg[0]), int(evt.msg[1]))
except Exception as e:
logger.debug('Garbage raised `{}` -- dropping.'.format(e))
self.evaluate_leaderboard()
elif evt.type == 'JOIN' and evt.group == self.sync_group:
should_announce = True
elif (evt.type == 'LEAVE' and evt.group == self.sync_group) or evt.type == 'EXIT':
self.remove_from_leaderboard(evt.peer_uuid)
self.evaluate_leaderboard()
if should_announce:
self.announce_clock_master_info()
if not self.has_been_synced and self.follower_service and self.follower_service.in_sync:
self.has_been_synced = 1.
self.announce_clock_master_info()
self.evaluate_leaderboard()
def update_leaderboard(self, uuid, name, rank, port):
for cs in self.leaderboard:
if cs.uuid == uuid:
if (cs.rank != rank) or (cs.port != port):
self.remove_from_leaderboard(cs.uuid)
break
else:
# no changes. Just leave as is
return
# clock service was not encountered before or has changed adding it to leaderboard
cs = Clock_Service(uuid, name, rank, port)
heappush(self.leaderboard, cs)
logger.debug('{} added'.format(cs))
def remove_from_leaderboard(self, uuid):
for cs in self.leaderboard:
if cs.uuid == uuid:
self.leaderboard.remove(cs)
logger.debug('{} removed'.format(cs))
break
def evaluate_leaderboard(self):
if not self.leaderboard:
logger.debug("nobody on the leader board.")
return
current_leader = self.leaderboard[0]
if self.discovery.uuid() != current_leader.uuid:
# we are not the leader!
leader_ep = self.discovery.peer_address(current_leader.uuid)
leader_addr = urlparse(leader_ep).netloc.split(':')[0]
if self.follower_service is None:
# make new follower
self.follower_service = Clock_Sync_Follower(leader_addr,
port=current_leader.port,
interval=10,
time_fn=self.get_time,
jump_fn=self.jump_time,
slew_fn=self.slew_time)
else:
# update follower_service
self.follower_service.host = leader_addr
self.follower_service.port = current_leader.port
return
# we are the leader
logger.debug("we are the leader")
if self.follower_service is not None:
self.follower_service.terminate()
self.follower_service = None
if not self.has_been_master:
self.has_been_master = 1.
logger.debug('Become clock master with rank {}'.format(self.rank))
self.announce_clock_master_info()
def announce_clock_master_info(self):
self.discovery.shout(self.sync_group, [repr(self.rank).encode(),
repr(self.master_service.port).encode()])
self.update_leaderboard(self.discovery.uuid(), self.node_name, self.rank, self.master_service.port)
@property
def rank(self):
return 4*self.base_bias + 2*self.has_been_master + self.has_been_synced + self.tie_breaker
def get_time(self):
return self.g_pool.get_timestamp()
def slew_time(self, offset):
self.g_pool.timebase.value += offset
def jump_time(self, offset):
ok_to_change = True
for p in self.g_pool.plugins:
if p.class_name == 'Recorder':
if p.running:
ok_to_change = False
logger.error("Request to change timebase during recording ignored. Turn off recording first.")
break
if ok_to_change:
self.slew_time(offset)
logger.info("Pupil Sync has adjusted the clock by {}s".format(offset))
return True
else:
return False
def restart_discovery(self, name):
if self.discovery:
if self.discovery.name() == name:
return
else:
self.discovery.leave(self.sync_group)
self.discovery.stop()
self.leaderboard = []
self.node_name = name or gethostname()
self.discovery = Pyre(self.node_name)
# Either joining network for the first time or rejoining the same group.
self.discovery.join(self.sync_group)
self.discovery.start()
self.announce_clock_master_info()
def change_sync_group(self, new_group_prefix):
if new_group_prefix != self.sync_group_prefix:
self.discovery.leave(self.sync_group)
self.leaderboard = []
if self.follower_service:
self.follower_service.terminate()
self.follower = None
self.sync_group_prefix = new_group_prefix
self.discovery.join(self.sync_group)
self.announce_clock_master_info()
def deinit_gui(self):
if self.menu:
self.g_pool.sidebar.remove(self.menu)
self.menu = None
def get_init_dict(self):
return {'node_name': self.node_name,
'sync_group_prefix': self.sync_group_prefix,
'base_bias': self.base_bias}
def cleanup(self):
self.deinit_gui()
self.discovery.leave(self.sync_group)
self.discovery.stop()
self.master_service.stop()
if self.follower_service:
self.follower_service.stop()
self.master_service = None
self.follower_service = None
self.discovery = None
class Agent:
"""
A class object that represents each app in the network"""
def __init__(self, name, ctx, group_name, cpu_clock_rate, experiment_name):
self.lock = threading.Lock()
self.cpu_clock_rate = cpu_clock_rate
self.cpu_load = random.random()
self.group_name = group_name
self.routing_table = None
self.name = name + str(os.getpid())
self.tasks = Queue(-1)
self.results = Queue(-1)
self.exp_name = experiment_name
self.task_duration_no_context = random.random()
# compute duration using cpu load, etc
self.task_duration_with_context = random.random()
#self.weights = 'rnn-model-attention-weights.h5'
#self.model = rnn_model()
# self.model._make_predict_function()
# self.model.load_weights(self.weights)
self.agent = Pyre(
name=self.name, ctx=ctx or zmq.Context.instance())
try:
self.agent.join(group_name)
self.agent.start()
except Exception as err:
logger.error(f'>>> Cant start node: {err}', exc_info=True)
def routing_table_setter(self, table):
self.lock.acquire()
try:
# create an ascending round robin routing principle
self.routing_table = cycle(
sorted(table.items(), key=lambda x: x[1]))
finally:
self.lock.release()
def add_task(self):
"""populates the task queue with new data for inference"""
logger.debug(f'>>> {threading.current_thread().name} started')
self.data = cycle(load_data(self.exp_name, 0))
count = 0
while count < 100:
task_dict = dict.fromkeys(
['input', 'target', 'task-type', 'task-uuid', 'task-owner-name', 'result', 'duration'], 0)
try:
input_data, target_data = next(self.data)
task_dict['input'] = input_data
task_dict['target'] = target_data
task_dict['task-type'] = 1
task_dict['task-uuid'] = self.agent.uuid()
task_dict['task-owner-name'] = self.agent.name()
task_dict['duration'] = time.time()
self.tasks.put(task_dict)
count += 1
except Exception as err:
logger.error(f'>>> Exception type: {err}', exc_info=True)
self.agent.leave(self.group_name)
self.agent.stop()
# Vary the frequency of input tasks
time.sleep(random.randint(1, 8))
def vary_cpu_load(self):
logger.debug(
f'>>> {threading.current_thread().name} thread started')
while True:
try:
self.lock.acquire()
self.cpu_load = random.random()
self.lock.release()
self.compute_duration_with_context()
except Exception as err:
logger.error(f'>>> Exception: {err}', exc_info=True)
time.sleep(random.randint(10, 40))
def compute_duration_with_context(self):
try:
self.lock.acquire()
cpu_load = self.cpu_load
task_duration_no_context = self.task_duration_no_context
self.task_duration_with_context = (
1 / task_duration_no_context) / (cpu_load * self.cpu_clock_rate)
self.lock.release()
except Exception as identifier:
logger.error(f'>>> Exception: {identifier}')
def compute_local(self, task):
"""argument is task"""
try:
task = task
task_data = task['input']
target = task['target']
uuid = task['task-uuid']
#predictions = self.model.predict(task_data, verbose=0)
#predictions = predictions.flatten()
# flatten the target
average = mean(task_data.flatten())
# window = 5
# errors = self.regression_error(predictions, target, window)
# mu, variance = np.mean(errors), np.var(errors)
# probabilities = self.chebyshev_probability(mu, variance, errors)
task['task-type'] = task['task-type'] + 1
if uuid == self.agent.uuid(): # put results in our queue if its our uuid
self.results.put(average)
self.lock.acquire()
self.task_duration_no_context = time.time() - task['duration']
self.lock.release()
self.compute_duration_with_context()
else:
task['result'] = average
data_byte = pickle.dumps(task, -1)
self.agent.whisper(uuid, data_byte)
logger.error(
f'>>> Results sent back to task owner peer: {task["task-owner-name"]}')
except Exception as identifier:
logger.error(f'>>> Exception type: {identifier}', exc_info=True)
self.agent.leave(self.group_name)
self.agent.stop() # clean up if there are issues.
def check_results(self):
logger.error(f'>>> {threading.current_thread().name} thread started')
while True:
try:
if not self.results.empty():
result = self.results.get()
if result <= 0.25:
logger.error(
f'>>> Critical anomaly detected: {result}')
elif result > 0.25 and result < 0.5:
logger.error(
f'>>> Severe anomaly detected: {result}')
elif result > 0.5 and result < 0.75:
logger.error(
f'>>> Serious anomaly detected: {result}')
else:
logger.error(f'>>> Mild anomaly detected: {result}')
except Exception as err:
logger.error(f'>>> Exception: {err}', exc_info=True)
self.agent.leave(self.group_name)
self.agent.stop()
def outbox(self, task, peer_uuid):
try:
task = pickle.dumps(task, -1)
self.agent.whisper(peer_uuid, task)
except Exception as identifier:
logger.error(f'>>> Exception: {identifier}',exc_info=True)
self.agent.leave(self.group_name)
self.agent.stop()
def num_of_peers(self, table):
seen = []
for peer in table:
if peer[0] in seen:
return len(seen)
else:
seen.append(peer[0])
def handle_task(self):
# decide if to compute locally or offload
logger.error(f'>>> {threading.current_thread().name} thread started')
while True:
try:
if not self.tasks.empty():
task = self.tasks.get()
self.lock.acquire()
local_duration = self.task_duration_with_context
table = self.routing_table
if table:
peer = next(table) # peer = (uuid, latency)
if peer[1] < local_duration:
self.outbox(task, peer[0])
logger.debug(f'>>> Task offloaded')
else:
num_of_peers = self.num_of_peers(table)
peer = self.search_table(
table, num_of_peers, local_duration)
if peer:
self.lock.release()
self.outbox(task, peer[0])
logger.debug(f'>>> Task offloaded')
else:
self.compute_local(task)
logger.debug(f'>>> Task computed locally')
else:
self.compute_local(task)
logger.debug(f'>>> Task computed locally')
except Exception as identifier:
logger.error(
f'>>> Exception type : {identifier}', exc_info=True)
self.agent.leave(self.group_name)
self.agent.stop() # stop if there are issues
time.sleep(random.randint(0, 3))
def search_table(self, table, num_of_peers, local_dur):
for id in range(num_of_peers):
peer = next(table)
if peer[1] < local_dur:
return peer
else:
return None
def inbox(self):
logger.error(f'>>> {threading.current_thread().name} thread started')
try:
events = self.agent.events() # works like charm
while True:
if events:
event = next(events)
logger.error(f'>>> MSG TYPE: {event.type}')
logger.error(f'>>> Sender Agent Name: {event.peer_name}')
if event.type == 'WHISPER':
msg = pickle.loads(event.msg[0])
if msg['task-type'] == 2:
result = msg['result']
self.results.put(result)
elif msg['task-type'] == 1: # peer sent us a task to execute
self.tasks.put(msg)
elif event.type == 'SHOUT': # message from the Access Point AP
msg = pickle.loads(event.msg[0])
if msg['msg-type'] == 'REQUEST':
msg['uuid'] = self.agent.uuid()
self.lock.acquire()
msg['processing-time'] = self.task_duration_with_context
self.lock.release()
msg_b = pickle.dumps(msg, -1)
self.agent.whisper(event.peer_uuid, msg_b)
elif msg['msg-type'] == 'UPDATE':
table = msg['table']
own_uuid = self.agent.uuid()
if own_uuid in table.keys():
# remove our own UUID to avoid offloading to ourselves
del table[own_uuid]
self.routing_table_setter(table)
except Exception as identifier:
logger.error(f'>>> Exception type: {identifier}', exc_info=True)
self.agent.leave(self.group_name)
self.agent.stop() # leave the cluster if you have issues
# compute the chebyshev probability
def chebyshev_probability(self, average, varianse, error_val):
probability = []
for val in error_val:
if val - average >= 1:
prob = varianse / ((val - average)**2)
probability.append(prob)
return probability
def regression_error(self, outcome, truth, window):
n_data = len(truth)
count = 0
errors = []
while count + window <= n_data:
error = [abs(y_pred - y_truth) for y_pred, y_truth in zip(
outcome[count:count + window], truth[count:count + window])]
errors.append(np.mean(error))
count += window
return errors
def run(self):
# start the threads here
t1 = threading.Thread(target=self.add_task, name='add task')
t2 = threading.Thread(target=self.vary_cpu_load, name='vary cpu load')
t3 = threading.Thread(target=self.check_results, name='check results')
t4 = threading.Thread(target=self.handle_task, name='handle task')
t5 = threading.Thread(target=self.inbox, name='inbox')
threads = [t1, t2, t3, t4, t5]
try:
for thread in threads:
thread.start()
except Exception as err:
logger.error(f'>>> Exception: {err}', exc_info=True)
self.agent.leave(self.group_name)
self.agent.stop()
