def worker(self):
"""
Worker thread that takes care of reading shared entries from ZK, and
handling master election.
"""
worker_cycle = 1.0
start = SCIONTime.get_time()
while self.run_flag.is_set():
sleep_interval(start, worker_cycle, "CS.worker cycle",
self._quiet_startup())
start = SCIONTime.get_time()
# Update IS_MASTER metric.
if self._labels:
IS_MASTER.labels(**self._labels).set(int(self.zk.have_lock()))
try:
self.zk.wait_connected()
self.trc_cache.process()
self.cc_cache.process()
self.drkey_cache.process()
# Try to become a master.
ret = self.zk.get_lock(lock_timeout=0, conn_timeout=0)
if ret: # Either got the lock, or already had it.
if ret == ZK_LOCK_SUCCESS:
logging.info("Became master")
self.trc_cache.expire(worker_cycle * 10)
self.cc_cache.expire(worker_cycle * 10)
self.drkey_cache.expire(worker_cycle * 10)
except ZkNoConnection:
logging.warning('worker(): ZkNoConnection')
pass
def _create_next_tree(self):
last_ttl_window = 0
ttl = self.config.revocation_tree_ttl
update_window = ttl // 3
while self.run_flag.is_set():
start = time.time()
cur_ttl_window = ConnectedHashTree.get_ttl_window(ttl)
time_to_sleep = ConnectedHashTree.time_until_next_window(
ttl) - update_window
if cur_ttl_window == last_ttl_window:
time_to_sleep += ttl
if time_to_sleep > 0:
sleep_interval(start, time_to_sleep, "BS._create_next_tree",
self._quiet_startup())
# at this point, there should be <= update_window
# seconds left in current ttl
logging.info("Started computing hashtree for next TTL window (%d)",
cur_ttl_window + 2)
last_ttl_window = ConnectedHashTree.get_ttl_window(ttl)
ht_start = time.time()
ifs = list(self.ifid2br.keys())
tree = ConnectedHashTree.get_next_tree(self.addr.isd_as, ifs,
self.hashtree_gen_key, ttl,
HashType.SHA256)
ht_end = time.time()
with self._hash_tree_lock:
self._next_tree = tree
logging.info(
"Finished computing hashtree for TTL window %d in %.3fs" %
(cur_ttl_window + 2, ht_end - ht_start))
def _handle_if_timeouts(self):
"""
Periodically checks each interface state and issues an if revocation, if
no keep-alive message was received for IFID_TOUT.
"""
if_id_last_revoked = defaultdict(int)
while self.run_flag.is_set():
start_time = time.time()
with self.ifid_state_lock:
to_revoke = []
for (if_id, if_state) in self.ifid_state.items():
cur_epoch = ConnectedHashTree.get_current_epoch()
if not if_state.is_expired() or (
if_state.is_revoked() and if_id_last_revoked[if_id] == cur_epoch):
# Either the interface hasn't timed out, or it's already revoked for this
# epoch
continue
if_id_last_revoked[if_id] = cur_epoch
if not if_state.is_revoked():
logging.info("IF %d went down.", if_id)
to_revoke.append(if_id)
if_state.revoke_if_expired()
self._issue_revocations(to_revoke)
sleep_interval(start_time, self.IF_TIMEOUT_INTERVAL,
"Handle IF timeouts")
def worker(self):
"""
Worker thread that takes care of reading shared paths from ZK, and
handling master election for core servers.
"""
worker_cycle = 1.0
start = SCIONTime.get_time()
was_master = False
while self.run_flag.is_set():
sleep_interval(start, worker_cycle, "cPS.worker cycle",
self._quiet_startup())
start = SCIONTime.get_time()
try:
self.zk.wait_connected()
self.path_cache.process()
self.rev_cache.process()
# Try to become a master.
is_master = self.zk.get_lock(lock_timeout=0, conn_timeout=0)
if is_master:
if not was_master:
logging.info("Became master")
self.path_cache.expire(self.config.propagation_time * 10)
self.rev_cache.expire(self.ZK_REV_OBJ_MAX_AGE)
was_master = True
else:
was_master = False
except ZkNoConnection:
logging.warning('worker(): ZkNoConnection')
pass
self._update_master()
self._propagate_and_sync()
def _handle_if_timeouts(self):
"""
Periodically checks each interface state and issues an if revocation, if
no keep-alive message was received for IFID_TOUT.
"""
while self.run_flag.is_set():
start_time = time.time()
for (if_id, if_state) in self.ifid_state.items():
# Check if interface has timed-out.
if if_state.is_expired():
logging.info("IF %d appears to be down.", if_id)
if if_id not in self.if2rev_tokens:
logging.error(
"Trying to issue revocation for " +
"non-existent if ID %d.", if_id)
continue
chain = self.if2rev_tokens[if_id]
self._issue_revocation(if_id, chain)
# Advance the hash chain for the corresponding IF.
try:
chain.move_to_next_element()
except HashChainExhausted:
# TODO(shitz): Add code to handle hash chain
# exhaustion.
logging.warning("HashChain for IF %s is exhausted.")
if_state.revoke_if_expired()
sleep_interval(start_time, self.IF_TIMEOUT_INTERVAL,
"Handle IF timeouts")
def _create_next_tree(self):
last_ttl_window = 0
while self.run_flag.is_set():
start = time.time()
cur_ttl_window = ConnectedHashTree.get_ttl_window()
time_to_sleep = (ConnectedHashTree.get_time_till_next_ttl() -
HASHTREE_UPDATE_WINDOW)
if cur_ttl_window == last_ttl_window:
time_to_sleep += HASHTREE_TTL
if time_to_sleep > 0:
sleep_interval(start, time_to_sleep, "BS._create_next_tree",
self._quiet_startup())
# at this point, there should be <= HASHTREE_UPDATE_WINDOW
# seconds left in current ttl
logging.info("Started computing hashtree for next TTL window (%d)",
cur_ttl_window + 2)
last_ttl_window = ConnectedHashTree.get_ttl_window()
ht_start = time.time()
ifs = list(self.ifid2br.keys())
tree = ConnectedHashTree.get_next_tree(self.addr.isd_as, ifs,
self.hashtree_gen_key,
HashType.SHA256)
ht_end = time.time()
with self._hash_tree_lock:
self._next_tree = tree
logging.info(
"Finished computing hashtree for TTL window %d in %.3fs" %
(cur_ttl_window + 2, ht_end - ht_start))
def _check_trc_cert_reqs(self):
check_cyle = 1.0
while self.run_flag.is_set():
start = time.time()
self._check_cert_reqs()
self._check_trc_reqs()
sleep_interval(start, check_cyle, "Elem._check_trc_cert_reqs cycle")
def worker(self):
start = time.time()
while self.run_flag.is_set():
sleep_interval(start, self.MONITOR_INTERVAL,
"LogMonitor.worker sleep", self._quiet_startup())
start = time.time()
self.ask_for_roots()
def worker(self):
"""
Worker thread that takes care of reading shared paths from ZK, and
handling master election for core servers.
"""
worker_cycle = 1.0
start = SCIONTime.get_time()
while self.run_flag.is_set():
sleep_interval(start, worker_cycle, "cPS.worker cycle",
self._quiet_startup())
start = SCIONTime.get_time()
try:
self.zk.wait_connected()
self.path_cache.process()
self.rev_cache.process()
# Try to become a master.
ret = self.zk.get_lock(lock_timeout=0, conn_timeout=0)
if ret: # Either got the lock, or already had it.
if ret == ZK_LOCK_SUCCESS:
logging.info("Became master")
self.path_cache.expire(self.config.propagation_time * 10)
self.rev_cache.expire(self.ZK_REV_OBJ_MAX_AGE)
except ZkNoConnection:
logging.warning('worker(): ZkNoConnection')
pass
self._update_master()
self._propagate_and_sync()
self._handle_pending_requests()
self._update_metrics()
def worker(self):
# Cycle time should be << SIBRA_TICK, as it determines how often
# reservations are potentially renewed, and the expiration of old
# reservation blocks.
worker_cycle = 1.0
start = SCIONTime.get_time()
while self.run_flag.is_set():
sleep_interval(start, worker_cycle, "SB.worker cycle")
start = SCIONTime.get_time()
with self.lock:
self.manage_steady_paths()
def worker(self):
start = time.time()
last_update = start
while self.run_flag.is_set():
sleep_interval(start, self.WORKER_INTERVAL, "LogServer.worker sleep",
self._quiet_startup())
if time.time() - last_update > self.UPDATE_INTERVAL:
self.update()
last_update = time.time()
self.handle_scps()
start = time.time()
def worker(self):
"""
Worker thread that takes care of reading shared PCBs from ZK, and
propagating PCBS/registering paths when master.
"""
last_propagation = last_registration = 0
last_ttl_window = ConnectedHashTree.get_ttl_window(
self.config.revocation_tree_ttl)
worker_cycle = 1.0
start = time.time()
while self.run_flag.is_set():
sleep_interval(start, worker_cycle, "BS.worker cycle",
self._quiet_startup())
start = time.time()
# Update IS_MASTER metric.
if self._labels:
IS_MASTER.labels(**self._labels).set(int(self.zk.have_lock()))
try:
self.zk.wait_connected()
self.pcb_cache.process()
self.revobjs_cache.process()
self.handle_rev_objs()
cur_ttl_window = ConnectedHashTree.get_ttl_window(
self.config.revocation_tree_ttl)
if cur_ttl_window != last_ttl_window:
self._maintain_hash_tree()
last_ttl_window = cur_ttl_window
ret = self.zk.get_lock(lock_timeout=0, conn_timeout=0)
if not ret: # Failed to get the lock
continue
elif ret == ZK_LOCK_SUCCESS:
logging.info("Became master")
self._became_master()
self.pcb_cache.expire(self.config.propagation_time * 10)
self.revobjs_cache.expire(self.ZK_REV_OBJ_MAX_AGE)
except ZkNoConnection:
continue
now = time.time()
if now - last_propagation >= self.config.propagation_time:
self.handle_pcbs_propagation()
last_propagation = now
if (self.config.registers_paths and
now - last_registration >= self.config.registration_time):
try:
self.register_segments()
except SCIONKeyError as e:
logging.error("Error while registering segments: %s", e)
pass
last_registration = now
def worker(self):
"""
Worker thread that takes care of reading shared PCBs from ZK, and
propagating PCBS/registering paths when master.
"""
last_propagation = last_registration = 0
last_ttl_window = ConnectedHashTree.get_ttl_window()
worker_cycle = 1.0
was_master = False
start = time.time()
while self.run_flag.is_set():
sleep_interval(start, worker_cycle, "BS.worker cycle",
self._quiet_startup())
start = time.time()
try:
self.process_pcb_queue()
self.handle_unverified_beacons()
self.zk.wait_connected()
self.pcb_cache.process()
self.revobjs_cache.process()
self.handle_rev_objs()
cur_ttl_window = ConnectedHashTree.get_ttl_window()
if cur_ttl_window != last_ttl_window:
self._maintain_hash_tree()
last_ttl_window = cur_ttl_window
if not self.zk.get_lock(lock_timeout=0, conn_timeout=0):
was_master = False
continue
if not was_master:
self._became_master()
was_master = True
self.pcb_cache.expire(self.config.propagation_time * 10)
self.revobjs_cache.expire(self.ZK_REV_OBJ_MAX_AGE)
except ZkNoConnection:
continue
now = time.time()
if now - last_propagation >= self.config.propagation_time:
self.handle_pcbs_propagation()
last_propagation = now
if (self.config.registers_paths and
now - last_registration >= self.config.registration_time):
try:
self.register_segments()
except SCIONKeyError as e:
logging.error("Register_segments: %s", e)
pass
last_registration = now
def request_ifstates(self):
"""
Periodically request interface states from the BS.
"""
pld = IFStateRequest.from_values()
while self.run_flag.is_set():
start_time = SCIONTime.get_time()
logging.info("Sending IFStateRequest for all interfaces.")
for bs in self.topology.beacon_servers:
req = self._build_packet(bs.addr, dst_port=bs.port,
payload=pld.copy())
self.send(req, bs.addr, SCION_UDP_EH_DATA_PORT)
sleep_interval(start_time, self.IFSTATE_REQ_INTERVAL,
"request_ifstates")
def _send_ifid_updates(self):
start = time.time()
while self.run_flag.is_set():
sleep_interval(start, self.IFID_INTERVAL,
"BS._send_ifid_updates cycle")
start = time.time()
# only master sends keep-alive messages
if not self.zk.have_lock():
continue
# send keep-alives on all known BR interfaces
for ifid in self.ifid2br:
br = self.ifid2br[ifid]
br_addr, br_port = br.int_addrs.public[0]
meta = self._build_meta(host=br_addr, port=br_port)
self.send_meta(CtrlPayload(IFIDPayload.from_values(ifid)),
meta, (meta.host, meta.port))
def _handle_if_timeouts(self):
"""
Periodically checks each interface state and issues an IF revocation, if
no keep-alive message was received for IFID_TOUT.
"""
while self.run_flag.is_set():
start_time = time.time()
with self.ifid_state_lock:
to_revoke = []
for (ifid, if_state) in self.ifid_state.items():
if self._labels:
metric = IF_STATE.labels(ifid=ifid, **self._labels)
if if_state.is_active():
metric.set(0)
elif if_state.is_revoked():
metric.set(1)
else:
metric.set(2)
if not if_state.is_expired():
# Interface hasn't timed out
self.if_revocations.pop(ifid, None)
continue
srev_info = self.if_revocations.get(ifid, None)
if if_state.is_revoked() and srev_info:
# Interface is revoked until the revocation time plus the revocation TTL,
# we want to issue a new revocation REVOCATION_OVERLAP seconds
# before it is expired
rev_info = srev_info.rev_info()
if (rev_info.p.timestamp + rev_info.p.ttl -
self.REVOCATION_OVERLAP > start_time):
# Interface has already been revoked within the REVOCATION_TTL -
# REVOCATION_OVERLAP period
continue
if not if_state.is_revoked():
logging.info("IF %d went down.", ifid)
to_revoke.append(ifid)
if_state.revoke_if_expired()
if to_revoke:
self._issue_revocations(to_revoke)
sleep_interval(start_time, self.IF_TIMEOUT_INTERVAL,
"Handle IF timeouts")
def _handle_if_timeouts(self):
"""
Periodically checks each interface state and issues an if revocation, if
no keep-alive message was received for IFID_TOUT.
"""
if_id_last_revoked = defaultdict(int)
while self.run_flag.is_set():
start_time = time.time()
with self.ifid_state_lock:
for (if_id, if_state) in self.ifid_state.items():
cur_epoch = ConnectedHashTree.get_current_epoch()
# Check if interface has timed-out.
if ((if_state.is_expired() or if_state.is_revoked())
and (if_id_last_revoked[if_id] != cur_epoch)):
if_id_last_revoked[if_id] = cur_epoch
if not if_state.is_revoked():
logging.info("IF %d appears to be down.", if_id)
self._issue_revocation(if_id)
if_state.revoke_if_expired()
sleep_interval(start_time, self.IF_TIMEOUT_INTERVAL,
"Handle IF timeouts")
def worker(self):
"""
Worker thread that takes care of reading shared entries from ZK, and
handling master election.
"""
worker_cycle = 1.0
start = SCIONTime.get_time()
while self.run_flag.is_set():
sleep_interval(start, worker_cycle, "CS.worker cycle",
self._quiet_startup())
start = SCIONTime.get_time()
try:
self.zk.wait_connected()
self.trc_cache.process()
self.cc_cache.process()
# Try to become a master.
if self.zk.get_lock(lock_timeout=0, conn_timeout=0):
self.trc_cache.expire(worker_cycle * 10)
self.cc_cache.expire(worker_cycle * 10)
except ZkNoConnection:
logging.warning('worker(): ZkNoConnection')
pass
def _send_ifid_updates(self):
start = time.time()
while self.run_flag.is_set():
sleep_interval(start, self.IFID_INTERVAL,
"BS._send_ifid_updates cycle")
start = time.time()
# only master sends keep-alive messages
if not self.zk.have_lock():
continue
# send keep-alives on all known BR interfaces
for ifid in self.ifid2br:
br = self.ifid2br[ifid]
br_addr, br_port = br.int_addrs.public
one_hop_path = self._create_one_hop_path(ifid)
meta = self._build_meta(ia=br.interfaces[ifid].isd_as,
host=SVCType.BS_M,
path=one_hop_path,
one_hop=True)
self.send_meta(CtrlPayload(IFIDPayload.from_values(ifid)),
meta, (br_addr, br_port))
def test_basic(self, warning, time_, sleep_):
time_.return_value = 3
sleep_interval(3, 2, "desc")
time_.assert_called_once_with()
ntools.eq_(warning.call_count, 0)
sleep_.assert_called_once_with(2)
def test_zero(self, warning, time_, sleep_):
time_.return_value = 3
sleep_interval(0, 2, "desc")
ntools.eq_(warning.call_count, 1)
sleep_.assert_called_once_with(0)
def sibra_worker(self):
while self.run_flag.is_set():
start_time = SCIONTime.get_time()
self.sibra_state.update_tick()
sleep_interval(start_time, 1.0, "sibra_worker")
