def worker(self): """ Worker thread that takes care of reading shared entries from ZK, and handling master election. """ worker_cycle = 1.0 start = SCIONTime.get_time() while self.run_flag.is_set(): sleep_interval(start, worker_cycle, "CS.worker cycle", self._quiet_startup()) start = SCIONTime.get_time() # Update IS_MASTER metric. if self._labels: IS_MASTER.labels(**self._labels).set(int(self.zk.have_lock())) try: self.zk.wait_connected() self.trc_cache.process() self.cc_cache.process() self.drkey_cache.process() # Try to become a master. ret = self.zk.get_lock(lock_timeout=0, conn_timeout=0) if ret: # Either got the lock, or already had it. if ret == ZK_LOCK_SUCCESS: logging.info("Became master") self.trc_cache.expire(worker_cycle * 10) self.cc_cache.expire(worker_cycle * 10) self.drkey_cache.expire(worker_cycle * 10) except ZkNoConnection: logging.warning('worker(): ZkNoConnection') pass
def _create_next_tree(self): last_ttl_window = 0 ttl = self.config.revocation_tree_ttl update_window = ttl // 3 while self.run_flag.is_set(): start = time.time() cur_ttl_window = ConnectedHashTree.get_ttl_window(ttl) time_to_sleep = ConnectedHashTree.time_until_next_window( ttl) - update_window if cur_ttl_window == last_ttl_window: time_to_sleep += ttl if time_to_sleep > 0: sleep_interval(start, time_to_sleep, "BS._create_next_tree", self._quiet_startup()) # at this point, there should be <= update_window # seconds left in current ttl logging.info("Started computing hashtree for next TTL window (%d)", cur_ttl_window + 2) last_ttl_window = ConnectedHashTree.get_ttl_window(ttl) ht_start = time.time() ifs = list(self.ifid2br.keys()) tree = ConnectedHashTree.get_next_tree(self.addr.isd_as, ifs, self.hashtree_gen_key, ttl, HashType.SHA256) ht_end = time.time() with self._hash_tree_lock: self._next_tree = tree logging.info( "Finished computing hashtree for TTL window %d in %.3fs" % (cur_ttl_window + 2, ht_end - ht_start))
def _handle_if_timeouts(self): """ Periodically checks each interface state and issues an if revocation, if no keep-alive message was received for IFID_TOUT. """ if_id_last_revoked = defaultdict(int) while self.run_flag.is_set(): start_time = time.time() with self.ifid_state_lock: to_revoke = [] for (if_id, if_state) in self.ifid_state.items(): cur_epoch = ConnectedHashTree.get_current_epoch() if not if_state.is_expired() or ( if_state.is_revoked() and if_id_last_revoked[if_id] == cur_epoch): # Either the interface hasn't timed out, or it's already revoked for this # epoch continue if_id_last_revoked[if_id] = cur_epoch if not if_state.is_revoked(): logging.info("IF %d went down.", if_id) to_revoke.append(if_id) if_state.revoke_if_expired() self._issue_revocations(to_revoke) sleep_interval(start_time, self.IF_TIMEOUT_INTERVAL, "Handle IF timeouts")
def worker(self): """ Worker thread that takes care of reading shared paths from ZK, and handling master election for core servers. """ worker_cycle = 1.0 start = SCIONTime.get_time() was_master = False while self.run_flag.is_set(): sleep_interval(start, worker_cycle, "cPS.worker cycle", self._quiet_startup()) start = SCIONTime.get_time() try: self.zk.wait_connected() self.path_cache.process() self.rev_cache.process() # Try to become a master. is_master = self.zk.get_lock(lock_timeout=0, conn_timeout=0) if is_master: if not was_master: logging.info("Became master") self.path_cache.expire(self.config.propagation_time * 10) self.rev_cache.expire(self.ZK_REV_OBJ_MAX_AGE) was_master = True else: was_master = False except ZkNoConnection: logging.warning('worker(): ZkNoConnection') pass self._update_master() self._propagate_and_sync()
def _handle_if_timeouts(self): """ Periodically checks each interface state and issues an if revocation, if no keep-alive message was received for IFID_TOUT. """ while self.run_flag.is_set(): start_time = time.time() for (if_id, if_state) in self.ifid_state.items(): # Check if interface has timed-out. if if_state.is_expired(): logging.info("IF %d appears to be down.", if_id) if if_id not in self.if2rev_tokens: logging.error( "Trying to issue revocation for " + "non-existent if ID %d.", if_id) continue chain = self.if2rev_tokens[if_id] self._issue_revocation(if_id, chain) # Advance the hash chain for the corresponding IF. try: chain.move_to_next_element() except HashChainExhausted: # TODO(shitz): Add code to handle hash chain # exhaustion. logging.warning("HashChain for IF %s is exhausted.") if_state.revoke_if_expired() sleep_interval(start_time, self.IF_TIMEOUT_INTERVAL, "Handle IF timeouts")
def _create_next_tree(self): last_ttl_window = 0 while self.run_flag.is_set(): start = time.time() cur_ttl_window = ConnectedHashTree.get_ttl_window() time_to_sleep = (ConnectedHashTree.get_time_till_next_ttl() - HASHTREE_UPDATE_WINDOW) if cur_ttl_window == last_ttl_window: time_to_sleep += HASHTREE_TTL if time_to_sleep > 0: sleep_interval(start, time_to_sleep, "BS._create_next_tree", self._quiet_startup()) # at this point, there should be <= HASHTREE_UPDATE_WINDOW # seconds left in current ttl logging.info("Started computing hashtree for next TTL window (%d)", cur_ttl_window + 2) last_ttl_window = ConnectedHashTree.get_ttl_window() ht_start = time.time() ifs = list(self.ifid2br.keys()) tree = ConnectedHashTree.get_next_tree(self.addr.isd_as, ifs, self.hashtree_gen_key, HashType.SHA256) ht_end = time.time() with self._hash_tree_lock: self._next_tree = tree logging.info( "Finished computing hashtree for TTL window %d in %.3fs" % (cur_ttl_window + 2, ht_end - ht_start))
def _check_trc_cert_reqs(self): check_cyle = 1.0 while self.run_flag.is_set(): start = time.time() self._check_cert_reqs() self._check_trc_reqs() sleep_interval(start, check_cyle, "Elem._check_trc_cert_reqs cycle")
def worker(self): start = time.time() while self.run_flag.is_set(): sleep_interval(start, self.MONITOR_INTERVAL, "LogMonitor.worker sleep", self._quiet_startup()) start = time.time() self.ask_for_roots()
def worker(self): """ Worker thread that takes care of reading shared paths from ZK, and handling master election for core servers. """ worker_cycle = 1.0 start = SCIONTime.get_time() while self.run_flag.is_set(): sleep_interval(start, worker_cycle, "cPS.worker cycle", self._quiet_startup()) start = SCIONTime.get_time() try: self.zk.wait_connected() self.path_cache.process() self.rev_cache.process() # Try to become a master. ret = self.zk.get_lock(lock_timeout=0, conn_timeout=0) if ret: # Either got the lock, or already had it. if ret == ZK_LOCK_SUCCESS: logging.info("Became master") self.path_cache.expire(self.config.propagation_time * 10) self.rev_cache.expire(self.ZK_REV_OBJ_MAX_AGE) except ZkNoConnection: logging.warning('worker(): ZkNoConnection') pass self._update_master() self._propagate_and_sync() self._handle_pending_requests() self._update_metrics()
def worker(self): # Cycle time should be << SIBRA_TICK, as it determines how often # reservations are potentially renewed, and the expiration of old # reservation blocks. worker_cycle = 1.0 start = SCIONTime.get_time() while self.run_flag.is_set(): sleep_interval(start, worker_cycle, "SB.worker cycle") start = SCIONTime.get_time() with self.lock: self.manage_steady_paths()
def worker(self): start = time.time() last_update = start while self.run_flag.is_set(): sleep_interval(start, self.WORKER_INTERVAL, "LogServer.worker sleep", self._quiet_startup()) if time.time() - last_update > self.UPDATE_INTERVAL: self.update() last_update = time.time() self.handle_scps() start = time.time()
def worker(self): """ Worker thread that takes care of reading shared PCBs from ZK, and propagating PCBS/registering paths when master. """ last_propagation = last_registration = 0 last_ttl_window = ConnectedHashTree.get_ttl_window( self.config.revocation_tree_ttl) worker_cycle = 1.0 start = time.time() while self.run_flag.is_set(): sleep_interval(start, worker_cycle, "BS.worker cycle", self._quiet_startup()) start = time.time() # Update IS_MASTER metric. if self._labels: IS_MASTER.labels(**self._labels).set(int(self.zk.have_lock())) try: self.zk.wait_connected() self.pcb_cache.process() self.revobjs_cache.process() self.handle_rev_objs() cur_ttl_window = ConnectedHashTree.get_ttl_window( self.config.revocation_tree_ttl) if cur_ttl_window != last_ttl_window: self._maintain_hash_tree() last_ttl_window = cur_ttl_window ret = self.zk.get_lock(lock_timeout=0, conn_timeout=0) if not ret: # Failed to get the lock continue elif ret == ZK_LOCK_SUCCESS: logging.info("Became master") self._became_master() self.pcb_cache.expire(self.config.propagation_time * 10) self.revobjs_cache.expire(self.ZK_REV_OBJ_MAX_AGE) except ZkNoConnection: continue now = time.time() if now - last_propagation >= self.config.propagation_time: self.handle_pcbs_propagation() last_propagation = now if (self.config.registers_paths and now - last_registration >= self.config.registration_time): try: self.register_segments() except SCIONKeyError as e: logging.error("Error while registering segments: %s", e) pass last_registration = now
def worker(self): """ Worker thread that takes care of reading shared PCBs from ZK, and propagating PCBS/registering paths when master. """ last_propagation = last_registration = 0 last_ttl_window = ConnectedHashTree.get_ttl_window() worker_cycle = 1.0 was_master = False start = time.time() while self.run_flag.is_set(): sleep_interval(start, worker_cycle, "BS.worker cycle", self._quiet_startup()) start = time.time() try: self.process_pcb_queue() self.handle_unverified_beacons() self.zk.wait_connected() self.pcb_cache.process() self.revobjs_cache.process() self.handle_rev_objs() cur_ttl_window = ConnectedHashTree.get_ttl_window() if cur_ttl_window != last_ttl_window: self._maintain_hash_tree() last_ttl_window = cur_ttl_window if not self.zk.get_lock(lock_timeout=0, conn_timeout=0): was_master = False continue if not was_master: self._became_master() was_master = True self.pcb_cache.expire(self.config.propagation_time * 10) self.revobjs_cache.expire(self.ZK_REV_OBJ_MAX_AGE) except ZkNoConnection: continue now = time.time() if now - last_propagation >= self.config.propagation_time: self.handle_pcbs_propagation() last_propagation = now if (self.config.registers_paths and now - last_registration >= self.config.registration_time): try: self.register_segments() except SCIONKeyError as e: logging.error("Register_segments: %s", e) pass last_registration = now
def request_ifstates(self): """ Periodically request interface states from the BS. """ pld = IFStateRequest.from_values() while self.run_flag.is_set(): start_time = SCIONTime.get_time() logging.info("Sending IFStateRequest for all interfaces.") for bs in self.topology.beacon_servers: req = self._build_packet(bs.addr, dst_port=bs.port, payload=pld.copy()) self.send(req, bs.addr, SCION_UDP_EH_DATA_PORT) sleep_interval(start_time, self.IFSTATE_REQ_INTERVAL, "request_ifstates")
def _send_ifid_updates(self): start = time.time() while self.run_flag.is_set(): sleep_interval(start, self.IFID_INTERVAL, "BS._send_ifid_updates cycle") start = time.time() # only master sends keep-alive messages if not self.zk.have_lock(): continue # send keep-alives on all known BR interfaces for ifid in self.ifid2br: br = self.ifid2br[ifid] br_addr, br_port = br.int_addrs.public[0] meta = self._build_meta(host=br_addr, port=br_port) self.send_meta(CtrlPayload(IFIDPayload.from_values(ifid)), meta, (meta.host, meta.port))
def _handle_if_timeouts(self): """ Periodically checks each interface state and issues an IF revocation, if no keep-alive message was received for IFID_TOUT. """ while self.run_flag.is_set(): start_time = time.time() with self.ifid_state_lock: to_revoke = [] for (ifid, if_state) in self.ifid_state.items(): if self._labels: metric = IF_STATE.labels(ifid=ifid, **self._labels) if if_state.is_active(): metric.set(0) elif if_state.is_revoked(): metric.set(1) else: metric.set(2) if not if_state.is_expired(): # Interface hasn't timed out self.if_revocations.pop(ifid, None) continue srev_info = self.if_revocations.get(ifid, None) if if_state.is_revoked() and srev_info: # Interface is revoked until the revocation time plus the revocation TTL, # we want to issue a new revocation REVOCATION_OVERLAP seconds # before it is expired rev_info = srev_info.rev_info() if (rev_info.p.timestamp + rev_info.p.ttl - self.REVOCATION_OVERLAP > start_time): # Interface has already been revoked within the REVOCATION_TTL - # REVOCATION_OVERLAP period continue if not if_state.is_revoked(): logging.info("IF %d went down.", ifid) to_revoke.append(ifid) if_state.revoke_if_expired() if to_revoke: self._issue_revocations(to_revoke) sleep_interval(start_time, self.IF_TIMEOUT_INTERVAL, "Handle IF timeouts")
def _handle_if_timeouts(self): """ Periodically checks each interface state and issues an if revocation, if no keep-alive message was received for IFID_TOUT. """ if_id_last_revoked = defaultdict(int) while self.run_flag.is_set(): start_time = time.time() with self.ifid_state_lock: for (if_id, if_state) in self.ifid_state.items(): cur_epoch = ConnectedHashTree.get_current_epoch() # Check if interface has timed-out. if ((if_state.is_expired() or if_state.is_revoked()) and (if_id_last_revoked[if_id] != cur_epoch)): if_id_last_revoked[if_id] = cur_epoch if not if_state.is_revoked(): logging.info("IF %d appears to be down.", if_id) self._issue_revocation(if_id) if_state.revoke_if_expired() sleep_interval(start_time, self.IF_TIMEOUT_INTERVAL, "Handle IF timeouts")
def worker(self): """ Worker thread that takes care of reading shared entries from ZK, and handling master election. """ worker_cycle = 1.0 start = SCIONTime.get_time() while self.run_flag.is_set(): sleep_interval(start, worker_cycle, "CS.worker cycle", self._quiet_startup()) start = SCIONTime.get_time() try: self.zk.wait_connected() self.trc_cache.process() self.cc_cache.process() # Try to become a master. if self.zk.get_lock(lock_timeout=0, conn_timeout=0): self.trc_cache.expire(worker_cycle * 10) self.cc_cache.expire(worker_cycle * 10) except ZkNoConnection: logging.warning('worker(): ZkNoConnection') pass
def _send_ifid_updates(self): start = time.time() while self.run_flag.is_set(): sleep_interval(start, self.IFID_INTERVAL, "BS._send_ifid_updates cycle") start = time.time() # only master sends keep-alive messages if not self.zk.have_lock(): continue # send keep-alives on all known BR interfaces for ifid in self.ifid2br: br = self.ifid2br[ifid] br_addr, br_port = br.int_addrs.public one_hop_path = self._create_one_hop_path(ifid) meta = self._build_meta(ia=br.interfaces[ifid].isd_as, host=SVCType.BS_M, path=one_hop_path, one_hop=True) self.send_meta(CtrlPayload(IFIDPayload.from_values(ifid)), meta, (br_addr, br_port))
def test_basic(self, warning, time_, sleep_): time_.return_value = 3 sleep_interval(3, 2, "desc") time_.assert_called_once_with() ntools.eq_(warning.call_count, 0) sleep_.assert_called_once_with(2)
def test_zero(self, warning, time_, sleep_): time_.return_value = 3 sleep_interval(0, 2, "desc") ntools.eq_(warning.call_count, 1) sleep_.assert_called_once_with(0)
def sibra_worker(self): while self.run_flag.is_set(): start_time = SCIONTime.get_time() self.sibra_state.update_tick() sleep_interval(start_time, 1.0, "sibra_worker")