def wait_connected(self, timeout=None):
"""
Wait until there is a connection to Zookeeper. Log every 10s until a
connection is available.
:param float timeout:
Number of seconds to wait for a ZK connection. If ``None``, wait
forever.
:raises:
ZkNoConnection:
if there's no connection to ZK after timeout has expired.
"""
if self.is_connected():
return
logging.debug("Waiting for ZK connection")
start = SCIONTime.get_time()
total_time = 0.0
if timeout is None:
next_timeout = 10.0
while True:
if timeout is not None:
next_timeout = min(timeout - total_time, 10.0)
ret = self._connected.wait(timeout=next_timeout)
total_time = SCIONTime.get_time() - start
if ret:
logging.debug("ZK connection available after %.2fs",
total_time)
return
elif timeout is not None and total_time >= timeout:
logging.debug("ZK connection still unavailable after %.2fs",
total_time)
raise ZkNoConnection
else:
logging.debug("Still waiting for ZK connection (%.2fs so far)",
total_time)
def worker(self):
"""
Worker thread that takes care of reading shared entries from ZK, and
handling master election.
"""
worker_cycle = 1.0
start = SCIONTime.get_time()
while self.run_flag.is_set():
sleep_interval(start, worker_cycle, "CS.worker cycle",
self._quiet_startup())
start = SCIONTime.get_time()
# Update IS_MASTER metric.
if self._labels:
IS_MASTER.labels(**self._labels).set(int(self.zk.have_lock()))
try:
self.zk.wait_connected()
self.trc_cache.process()
self.cc_cache.process()
self.drkey_cache.process()
# Try to become a master.
ret = self.zk.get_lock(lock_timeout=0, conn_timeout=0)
if ret: # Either got the lock, or already had it.
if ret == ZK_LOCK_SUCCESS:
logging.info("Became master")
self.trc_cache.expire(worker_cycle * 10)
self.cc_cache.expire(worker_cycle * 10)
self.drkey_cache.expire(worker_cycle * 10)
except ZkNoConnection:
logging.warning('worker(): ZkNoConnection')
pass
def store(self, name, value):
"""
Store an entry in the cache.
:param str name: Name of the entry. E.g. ``"item01"``.
:param bytes value: The value of the entry.
:raises:
ZkNoConnection: if there's no connection to ZK.
"""
if not self._zk.is_connected():
raise ZkNoConnection
full_path = os.path.join(self._path, name)
# First, assume the entry already exists (the normal case)
try:
self._kazoo.set(full_path, value)
self._incoming_entries.append((name, SCIONTime.get_time()))
return
except NoNodeError:
pass
except (ConnectionLoss, SessionExpiredError):
raise ZkNoConnection from None
# Entry doesn't exist, so create it instead.
try:
self._kazoo.create(full_path, value, makepath=True)
self._incoming_entries.append((name, SCIONTime.get_time()))
return
except NodeExistsError:
# Entry was created between our check and our create, so assume that
# the contents are recent, and return without error.
pass
except (ConnectionLoss, SessionExpiredError):
raise ZkNoConnection from None
def worker(self):
"""
Worker thread that takes care of reading shared paths from ZK, and
handling master election for core servers.
"""
worker_cycle = 1.0
start = SCIONTime.get_time()
while self.run_flag.is_set():
sleep_interval(start, worker_cycle, "cPS.worker cycle",
self._quiet_startup())
start = SCIONTime.get_time()
try:
self.zk.wait_connected()
self.path_cache.process()
self.rev_cache.process()
# Try to become a master.
ret = self.zk.get_lock(lock_timeout=0, conn_timeout=0)
if ret: # Either got the lock, or already had it.
if ret == ZK_LOCK_SUCCESS:
logging.info("Became master")
self.path_cache.expire(self.config.propagation_time * 10)
self.rev_cache.expire(self.ZK_REV_OBJ_MAX_AGE)
except ZkNoConnection:
logging.warning('worker(): ZkNoConnection')
pass
self._update_master()
self._propagate_and_sync()
self._handle_pending_requests()
self._update_metrics()
def worker(self):
"""
Worker thread that takes care of reading shared paths from ZK, and
handling master election for core servers.
"""
worker_cycle = 1.0
start = SCIONTime.get_time()
was_master = False
while self.run_flag.is_set():
sleep_interval(start, worker_cycle, "cPS.worker cycle",
self._quiet_startup())
start = SCIONTime.get_time()
try:
self.zk.wait_connected()
self.path_cache.process()
self.rev_cache.process()
# Try to become a master.
is_master = self.zk.get_lock(lock_timeout=0, conn_timeout=0)
if is_master:
if not was_master:
logging.info("Became master")
self.path_cache.expire(self.config.propagation_time * 10)
self.rev_cache.expire(self.ZK_REV_OBJ_MAX_AGE)
was_master = True
else:
was_master = False
except ZkNoConnection:
logging.warning('worker(): ZkNoConnection')
pass
self._update_master()
self._propagate_and_sync()
def worker(self):
# Cycle time should be << SIBRA_TICK, as it determines how often
# reservations are potentially renewed, and the expiration of old
# reservation blocks.
worker_cycle = 1.0
start = SCIONTime.get_time()
while self.run_flag.is_set():
sleep_interval(start, worker_cycle, "SB.worker cycle")
start = SCIONTime.get_time()
with self.lock:
self.manage_steady_paths()
def _zk_write_rev(self, data):
hash_ = crypto_hash(data).hex()
try:
self.rev_cache.store("%s-%s" % (hash_, SCIONTime.get_time()), data)
except ZkNoConnection:
logging.warning("Unable to store revocation(s) in shared path: "
"no connection to ZK")
def get_paths(self, dst_ia, flags=(), flush=False):
"""Return a list of paths."""
logging.debug("Paths requested for ISDAS=%s, flags=%s, flush=%s",
dst_ia, flags, flush)
if flush:
logging.info("Flushing PathDBs.")
self._flush_path_dbs()
if self.addr.isd_as == dst_ia or (
self.addr.isd_as.any_as() == dst_ia and
self.topology.is_core_as):
# Either the destination is the local AS, or the destination is any
# core AS in this ISD, and the local AS is in the core
empty = SCIONPath()
empty_meta = FwdPathMeta.from_values(empty, [], self.topology.mtu)
return [empty_meta], SCIONDPathReplyError.OK
deadline = SCIONTime.get_time() + self.PATH_REQ_TOUT
e = threading.Event()
self.requests.put(((dst_ia, flags), e))
if not self._wait_for_events([e], deadline):
logging.error("Query timed out for %s", dst_ia)
return [], SCIONDPathReplyError.PS_TIMEOUT
paths = self.path_resolution(dst_ia, flags=flags)
error_code = (SCIONDPathReplyError.OK if paths
else SCIONDPathReplyError.NO_PATHS)
return paths, error_code
def update_fidelity(self, path_policy):
"""
Computes a path fidelity based on all path properties and considering
the corresponding weights, which are stored in the path policy.
:param dict path_policy: path policy.
"""
self.fidelity = 0
now = SCIONTime.get_time()
self.fidelity += (path_policy.property_weights['PeerLinks'] *
self.peer_links)
self.fidelity += (path_policy.property_weights['HopsLength'] /
self.hops_length)
self.fidelity += (path_policy.property_weights['Disjointness'] *
self.disjointness)
if now != 0:
self.fidelity += (path_policy.property_weights['LastSentTime'] *
(now - self.last_sent_time) / now)
self.fidelity += (path_policy.property_weights['LastSeenTime'] *
self.last_seen_time / now)
self.fidelity += (path_policy.property_weights['DelayTime'] /
self.delay_time)
self.fidelity += (path_policy.property_weights['ExpirationTime'] *
(self.expiration_time - now) / self.expiration_time)
self.fidelity += (path_policy.property_weights['GuaranteedBandwidth'] *
self.guaranteed_bandwidth)
self.fidelity += (path_policy.property_weights['AvailableBandwidth'] *
self.available_bandwidth)
self.fidelity += (path_policy.property_weights['TotalBandwidth'] *
self.total_bandwidth)
def _add_req(self, key, request):
self._expire_reqs(key)
if not self._check(key):
self._fetch(key, request)
self._req_map[key].append((SCIONTime.get_time(), request))
if self._labels:
REQS_PENDING.labels(**self._labels).inc()
def expire(self, ttl):
"""
Delete entries first seen more than `ttl` seconds ago.
:param float ttl:
Age (in seconds) after which cache entries should be removed.
:raises:
ZkNoConnection: if there's no connection to ZK.
ZkNoNodeError: if a node disappears unexpectedly.
"""
if not self._zk.is_connected():
raise ZkNoConnection
now = SCIONTime.get_time()
count = 0
for entry, ts in self._entries.items():
if now - ts > ttl:
full_path = os.path.join(self._path, entry)
count += 1
try:
self._kazoo.delete(full_path)
except NoNodeError:
# This shouldn't happen, so raise an exception if it does.
raise ZkNoNodeError
except (ConnectionLoss, SessionExpiredError):
raise ZkNoConnection from None
if count:
logging.debug("Expired %d old entries from %s", count, self._path)
def _create_one_hop_path(self, egress_if):
ts = int(SCIONTime.get_time())
info = InfoOpaqueField.from_values(ts, self.addr.isd_as[0], hops=2)
hf1 = HopOpaqueField.from_values(self.HOF_EXP_TIME, 0, egress_if)
hf1.set_mac(self.of_gen_key, ts, None)
# Return a path where second HF is empty.
return SCIONPath.from_values(info, [hf1, HopOpaqueField()])
def _check_property_ranges(self, pcb):
"""
Checks whether any of the path properties has a value outside the
predefined min-max range.
:param pcb: beacon to analyze.
:type pcb: :class:`PathSegment`
"""
def _check_range(name, actual):
range_ = self.property_ranges[name]
if not range_:
return
if (actual < range_[0] or actual > range_[1]):
reasons.append("%s: %d <= %d <= %d" %
(name, range_[0], actual, range_[1]))
reasons = []
_check_range("PeerLinks", pcb.get_n_peer_links())
_check_range("HopsLength", pcb.get_n_hops())
_check_range("DelayTime",
int(SCIONTime.get_time()) - pcb.get_timestamp())
_check_range("GuaranteedBandwidth", 10)
_check_range("AvailableBandwidth", 10)
_check_range("TotalBandwidth", 10)
return reasons
def handle_pcbs_propagation(self):
"""
Generate a new beacon or gets ready to forward the one received.
"""
timestamp = int(SCIONTime.get_time())
# Create beacon for downstream ASes.
down_iof = InfoOpaqueField.from_values(timestamp, self.addr.isd_as[0])
downstream_pcb = PathSegment.from_values(down_iof)
propagated_pcbs = self.propagate_downstream_pcb(downstream_pcb)
# Create beacon for core ASes.
core_iof = InfoOpaqueField.from_values(timestamp, self.addr.isd_as[0])
core_pcb = PathSegment.from_values(core_iof)
propagated = self.propagate_core_pcb(core_pcb)
for k, v in propagated.items():
propagated_pcbs[k].extend(v)
# Propagate received beacons. A core beacon server can only receive
# beacons from other core beacon servers.
beacons = []
with self._rev_seg_lock:
for ps in self.core_beacons.values():
beacons.extend(ps.get_best_segments())
for pcb in beacons:
propagated = self.propagate_core_pcb(pcb)
for k, v in propagated.items():
propagated_pcbs[k].extend(v)
self._log_propagations(propagated_pcbs)
def update(self, pcb, reverse=False):
"""
Insert path into database.
Return the result of the operation.
"""
first_ia = pcb.first_ia()
last_ia = pcb.last_ia()
if reverse:
first_ia, last_ia = last_ia, first_ia
if self._segment_ttl:
now = int(SCIONTime.get_time())
record = PathSegmentDBRecord(pcb, now + self._segment_ttl)
else:
record = PathSegmentDBRecord(pcb)
with self._lock:
recs = self._db(id=record.id, sibra=pcb.is_sibra())
assert len(recs) <= 1, "PathDB contains > 1 path with the same ID"
if not recs:
self._db.insert(
record, record.id, first_ia[0], first_ia[1],
last_ia[0], last_ia[1], pcb.is_sibra())
logging.debug("Added segment from %s to %s: %s",
first_ia, last_ia, pcb.short_desc())
return DBResult.ENTRY_ADDED
cur_rec = recs[0]['record']
if pcb.get_expiration_time() < cur_rec.pcb.get_expiration_time():
return DBResult.NONE
cur_rec.pcb = pcb
if self._segment_ttl:
cur_rec.exp_time = now + self._segment_ttl
else:
cur_rec.exp_time = pcb.get_expiration_time()
return DBResult.ENTRY_UPDATED
def _remove_expired_segments(self):
"""Remove candidates if their expiration_time is up."""
rec_ids = []
now = SCIONTime.get_time()
for candidate in self.candidates:
if candidate.expiration_time <= now:
rec_ids.append(candidate.id)
self.remove_segments(rec_ids)
def _zk_write(self, data):
hash_ = SHA256.new(data).hexdigest()
try:
self.path_cache.store("%s-%s" % (hash_, SCIONTime.get_time()),
data)
except ZkNoConnection:
logging.warning("Unable to store segment(s) in shared path: "
"no connection to ZK")
def _add_req(self, key, request):
self._req_map.setdefault(key, [])
self._expire_reqs(key)
if not self._check(key) and len(self._req_map[key]) == 0:
# Don't already have the answer, and there were no outstanding
# requests, so send a new query
self._fetch(key, request)
self._req_map[key].append((SCIONTime.get_time(), request))
def update(self, pcb):
"""
Update a candidate entry from a recent PCB.
"""
assert self.id == pcb.get_hops_hash(hex=True)
now = int(SCIONTime.get_time())
self.pcb = copy.deepcopy(pcb)
self.delay_time = now - pcb.get_timestamp()
self.last_seen_time = now
self.expiration_time = pcb.get_expiration_time()
def append_hop(self, isd_as, if_id, timestamp=None): # pragma: no cover
"""
Append hop's information as a new field in the extension.
"""
# Check whether
assert len(self.hops) < self._hdr_len
if timestamp is None:
# Truncate milliseconds to 2B
timestamp = int(SCIONTime.get_time() * 1000) % 2**16
self.hops.append((isd_as, if_id, timestamp))
def verify_hof(self, path, ingress=True):
"""Verify freshness and authentication of an opaque field."""
ts = path.get_iof().timestamp
hof = path.get_hof()
prev_hof = path.get_hof_ver(ingress=ingress)
if int(SCIONTime.get_time()) <= ts + hof.exp_time * EXP_TIME_UNIT:
if not hof.verify_mac(self.of_gen_key, ts, prev_hof):
raise SCIONOFVerificationError(hof, prev_hof)
else:
raise SCIONOFExpiredError(hof)
def _wait_for_events(self, events, deadline):
"""
Wait on a set of events, but only until the specified deadline. Returns
the number of events that happened while waiting.
"""
count = 0
for e in events:
if e.wait(max(0, deadline - SCIONTime.get_time())):
count += 1
return count
def _share_object(self, pld, is_trc):
"""
Share path segments (via ZK) with other path servers.
"""
pld_packed = pld.pack()
pld_hash = crypto_hash(pld_packed).hex()
try:
if is_trc:
self.trc_cache.store(
"%s-%s" % (pld_hash, SCIONTime.get_time()), pld_packed)
else:
self.cc_cache.store("%s-%s" % (pld_hash, SCIONTime.get_time()),
pld_packed)
except ZkNoConnection:
logging.warning("Unable to store %s in shared path: "
"no connection to ZK" % "TRC" if is_trc else "CC")
return
logging.debug("%s stored in ZK: %s" %
("TRC" if is_trc else "CC", pld_hash))
def _expire_reqs(self, key):
if key not in self._req_map:
return
now = SCIONTime.get_time()
count = 0
for ts, req in self._req_map[key][:]:
if now - ts > self._ttl:
count += 1
self._req_map[key].remove((ts, req))
if count:
logging.debug("Expired %d requests for %s", count, key)
def _update_disjointness_db(self):
"""
Update the disjointness database.
Based on the current time, update the disjointness database keeping
track of each path, AS, and interface previously sent.
"""
now = SCIONTime.get_time()
for k, v in self.disjointness.items():
self.disjointness[k] = v * math.exp(self.last_dj_update - now)
self.last_dj_update = now
def __contains__(self, key):
""" Return True if the dict has a key, else return False. """
try:
with self.lock:
item = OrderedDict.__getitem__(self, key)
if SCIONTime.get_time() - item[1] < self.max_age:
return True
else:
del self[key]
except KeyError:
pass
return False
def worker(self):
"""
Worker thread that takes care of reading shared entries from ZK, and
handling master election.
"""
worker_cycle = 1.0
start = SCIONTime.get_time()
while self.run_flag.is_set():
sleep_interval(start, worker_cycle, "CS.worker cycle",
self._quiet_startup())
start = SCIONTime.get_time()
try:
self.zk.wait_connected()
self.trc_cache.process()
self.cc_cache.process()
# Try to become a master.
if self.zk.get_lock(lock_timeout=0, conn_timeout=0):
self.trc_cache.expire(worker_cycle * 10)
self.cc_cache.expire(worker_cycle * 10)
except ZkNoConnection:
logging.warning('worker(): ZkNoConnection')
pass
def request_ifstates(self):
"""
Periodically request interface states from the BS.
"""
pld = IFStateRequest.from_values()
while self.run_flag.is_set():
start_time = SCIONTime.get_time()
logging.info("Sending IFStateRequest for all interfaces.")
for bs in self.topology.beacon_servers:
req = self._build_packet(bs.addr, dst_port=bs.port,
payload=pld.copy())
self.send(req, bs.addr, SCION_UDP_EH_DATA_PORT)
sleep_interval(start_time, self.IFSTATE_REQ_INTERVAL,
"request_ifstates")
def __getitem__(self, key, with_age=False):
""" Return the item of the dict.
Raises a KeyError if key is not in the map.
"""
with self.lock:
item = OrderedDict.__getitem__(self, key)
item_age = SCIONTime.get_time() - item[1]
if item_age < self.max_age:
if with_age:
return item[0], item_age
else:
return item[0]
else:
del self[key]
raise KeyError(key)
def _exp_call_records(self, recs):
"""Remove expired segments from the db."""
now = int(SCIONTime.get_time())
ret = []
expired = []
for r in recs:
if r['record'].exp_time < now:
expired.append(r)
logging.debug("Path-Segment expired: %s",
r['record'].pcb.short_desc())
continue
ret.append(r)
if expired:
self._db.delete(expired)
return ret
