def __remove_redundant_hosts_from_db():
to_delete = KnownHosts.select() \
.order_by(KnownHosts.last_connected.desc()) \
.offset(MAX_STORED_HOSTS)
KnownHosts.delete() \
.where(KnownHosts.id << to_delete) \
.execute()
def __sync_seeds(self, known_hosts=None):
if not known_hosts:
known_hosts = KnownHosts.select().where(KnownHosts.is_seed)
self.seeds = {(x.ip_address, x.port) for x in known_hosts if x.is_seed}
self.seeds.update(SEEDS)
ip_address = self.config_desc.seed_host
port = self.config_desc.seed_port
if ip_address and port:
self.seeds.add((ip_address, port))
def add_known_peer(self, node, ip_address, port):
is_seed = node.is_super_node() if node else False
try:
with db.transaction():
KnownHosts.delete().where((KnownHosts.ip_address == ip_address)
&
(KnownHosts.port == port)).execute()
KnownHosts.insert(ip_address=ip_address,
port=port,
last_connected=time.time(),
is_seed=is_seed).execute()
self.__remove_redundant_hosts_from_db()
self.__sync_seeds()
except Exception as err:
logger.error("Couldn't add known peer %r:%r : %s", ip_address,
port, err)
def get_performance_percentile_rank(perf: float, env_id: str) -> float:
# Hosts which don't support the given env at all shouldn't be counted
# even if perf equals 0. Therefore -1 is the default value.
hosts_perf = [
host.metadata['performance'].get(env_id, -1.0)
for host in KnownHosts.select() if 'performance' in host.metadata
]
if not hosts_perf:
logger.warning('Cannot compute percentile rank. No host '
'performance info is available')
return 1.0
rank = sum(1 for x in hosts_perf if x < perf) / len(hosts_perf)
logger.info(f'Performance for env `{env_id}`: rank({perf}) = {rank}')
return rank
def test_add_known_peer(self):
key_id = EllipticalKeysAuth(self.path, "TESTPRIV", "TESTPUB").get_key_id()
nominal_seeds = len(self.service.seeds)
node = Node(
'super_node', key_id,
pub_addr='1.2.3.4',
prv_addr='1.2.3.4',
pub_port=10000,
prv_port=10000
)
node.prv_addresses = [node.prv_addr, '172.1.2.3']
assert Node.is_super_node(node)
KnownHosts.delete().execute()
len_start = len(KnownHosts.select())
# insert one
self.service.add_known_peer(node, node.pub_addr, node.pub_port)
select_1 = KnownHosts.select()
len_1 = len(select_1)
last_conn_1 = select_1[0].last_connected
assert len_1 > len_start
# advance time
time.sleep(0.1)
# insert duplicate
self.service.add_known_peer(node, node.pub_addr, node.pub_port)
select_2 = KnownHosts.select()
len_2 = len(select_2)
assert len_2 == len_1
assert select_2[0].last_connected > last_conn_1
assert len(self.service.seeds) > nominal_seeds
# try to add more than max, we already have at least 1
pub_prefix = '2.2.3.'
prv_prefix = '172.1.2.'
for i in xrange(1, MAX_STORED_HOSTS + 6):
i_str = str(i)
pub = pub_prefix + i_str
prv = prv_prefix + i_str
n = Node(
i_str, key_id + i_str,
pub_addr=pub,
prv_addr=prv,
pub_port=10000,
prv_port=10000
)
self.service.add_known_peer(n, pub, n.prv_port)
assert len(KnownHosts.select()) == MAX_STORED_HOSTS
assert len(self.service.seeds) == nominal_seeds
def test_add_known_peer(self):
key_id = encode_hex(urandom(64))[2:]
nominal_seeds = len(self.service.seeds)
node = Node(node_name='super_node',
key=str(key_id),
pub_addr='1.2.3.4',
prv_addr='1.2.3.4',
pub_port=10000,
prv_port=10000)
node.prv_addresses = [node.prv_addr, '172.1.2.3']
assert Node.is_super_node(node)
KnownHosts.delete().execute()
len_start = len(KnownHosts.select())
# insert one
self.service.add_known_peer(node, node.pub_addr, node.pub_port)
select_1 = KnownHosts.select()
len_1 = len(select_1)
last_conn_1 = select_1[0].last_connected
assert len_1 > len_start
# advance time
time.sleep(0.1)
# insert duplicate
self.service.add_known_peer(node, node.pub_addr, node.pub_port)
select_2 = KnownHosts.select()
len_2 = len(select_2)
assert len_2 == len_1
assert select_2[0].last_connected > last_conn_1
assert len(self.service.seeds) > nominal_seeds
# try to add more than max, we already have at least 1
pub_prefix = '2.2.3.'
prv_prefix = '172.1.2.'
key_id_str = key_id
for i in range(1, MAX_STORED_HOSTS + 6):
i_str = str(i)
pub = pub_prefix + i_str
prv = prv_prefix + i_str
n = Node(node_name=i_str,
key=key_id_str + i_str,
pub_addr=pub,
prv_addr=prv,
pub_port=10000,
prv_port=10000)
self.service.add_known_peer(n, pub, n.prv_port)
assert len(KnownHosts.select()) == MAX_STORED_HOSTS
assert len(self.service.seeds) == nominal_seeds
def connect_to_network(self):
self.connect_to_seeds()
if not self.connect_to_known_hosts:
return
for host in KnownHosts.select().where(
KnownHosts.is_seed == False): # noqa
ip_address = host.ip_address
port = host.port
logger.debug("Connecting to {}:{}".format(ip_address, port))
try:
socket_address = tcpnetwork.SocketAddress(ip_address, port)
self.connect(socket_address)
except Exception as exc:
logger.error("Cannot connect to host {}:{}: {}".format(
ip_address, port, exc))
def _sync_seeds(self, known_hosts=None):
self.last_seeds_sync = time.time()
if not known_hosts:
known_hosts = KnownHosts.select().where(KnownHosts.is_seed)
def _resolve_hostname(host, port):
try:
port = int(port)
except ValueError:
logger.info(
"Invalid seed: %s:%s. Ignoring.",
host,
port,
)
return
if not (host and port):
logger.debug(
"Ignoring incomplete seed. host=%r port=%r",
host,
port,
)
return
try:
for addrinfo in socket.getaddrinfo(host, port):
yield addrinfo[4] # (ip, port)
except OSError as e:
logger.error(
"Can't resolve %s:%s. %s",
host,
port,
e,
)
self.seeds = set()
ip_address = self.config_desc.seed_host
port = self.config_desc.seed_port
for hostport in itertools.chain(
((kh.ip_address, kh.port) for kh in known_hosts if kh.is_seed),
self.bootstrap_seeds, ((ip_address, port), ),
(cs.split(':', 1) for cs in self.config_desc.seeds.split(None, ))):
self.seeds.update(_resolve_hostname(*hostport))
def add_known_peer(self, node, ip_address, port, metadata=None):
is_seed = node.is_super_node() if node else False
try:
with db.transaction():
host, _ = KnownHosts.get_or_create(
ip_address=ip_address,
port=port,
defaults={'is_seed': is_seed})
host.last_connected = time.time()
host.metadata = metadata or {}
host.save()
self.__remove_redundant_hosts_from_db()
self._sync_seeds()
except Exception as err:
logger.error("Couldn't add known peer %r:%r : %s", ip_address,
port, err)
def connect_to_network(self):
# pylint: disable=singleton-comparison
self.connect_to_seeds()
if not self.connect_to_known_hosts:
return
for host in KnownHosts.select() \
.where(KnownHosts.is_seed == False)\
.limit(self.config_desc.opt_peer_num): # noqa
ip_address = host.ip_address
port = host.port
logger.debug("Connecting to {}:{}".format(ip_address, port))
try:
socket_address = tcpnetwork.SocketAddress(ip_address, port)
self.connect(socket_address)
except Exception as exc:
logger.error("Cannot connect to host {}:{}: {}".format(
ip_address, port, exc))