def test_exchange_views(self):
p1 = PartialView("First IP", 4, 3)
p1.add_peer(PodDescriptor("172.0.1.6", 0))
p1.add_peer(PodDescriptor("172.0.1.3", 2))
p1.add_peer(PodDescriptor("172.0.1.5", 3))
p1.add_peer(PodDescriptor("Second IP", 5))
p2 = PartialView("Second IP", 4, 3)
p2.add_peer(PodDescriptor("172.0.1.3", 0))
p2.add_peer(PodDescriptor("172.0.1.5", 1))
p2.add_peer(PodDescriptor("172.0.1.2", 2))
p2.add_peer(PodDescriptor("172.0.1.1", 4))
########################
# P1 starts the exchange
########################
# 1) Increase by one the age of all neighbors
p1.increment()
# 2) Select neighbor Q with the highest age among all neighbors.
oldest = p1.get_oldest_peer()
# 3) Select l - 1 other random neighbors (meaning avoid oldest).
request = p1.select_neighbors_for_request(oldest)
# 4) Replace Q's entry with a new entry of age 0 and with P's address.
request.add_peer_ip(p1.ip, allow_self_ip=True)
self.assertTrue(request.is_full())
self.assertEqual(request.size, p1.shuffle_length)
################################################
# P2 receives neighbors and prepares a reply
################################################
reply = p2.select_neighbors_for_reply()
self.assertTrue(request.is_full())
self.assertEqual(request.size, p1.shuffle_length)
# Note that in p1 the oldest is p2
# p1 and p2 know two peers in common
# p2 does not have an entry with p1's ip
# p1.merge should:
# - Discard 172.0.1.3 and 172.0.1.5
# - Put in unknown list 172.0.1.2, 172.0.1.1
# 6) I remove the oldest peer from my view
p1.remove_peer(oldest)
p1.merge(request, reply)
self.assertTrue(p1.is_full())
for peer in reply.get_peer_list():
self.assertTrue(p1.contains(peer))
self.assertLessEqual(self.partialView.size, self.partialView.limit)
class Cyclon(object):
def __init__(self):
self.ip = os.environ['MY_POD_IP']
self.k8s = KubernetesClient()
self.api_version = 'v1'
self.partialView = PartialView(self.ip)
self.bootstrap()
def bootstrap(self):
self.bootstrap_exponential_backoff(5, 5)
self.schedule_change(15, 15)
def bootstrap_exponential_backoff(self, initial_delay, delay):
logger.debug("Init", ip=self.ip, partialView=self.partialView)
time.sleep(initial_delay)
app_name = os.environ['APP']
attempt = 1
ips = self.k8s.list_pods_ips_by_field_selector(
label_selector="app=" + app_name,
field_selector="status.phase=Running")
logger.debug("Bootstrapping", running_pods=ips, attempt=attempt)
# Exponential backoff starts in case the number of running pods is lower than the partialView's limit.
# TODO: Did I consider also that some pods might not be ready yet?
# TODO: Consider that there is no need to have at least self.partialView.limit peers ready to start!
# TODO: There can be peers running with an initial partial view of size < self.partialView.limit
while len(ips) <= self.partialView.limit:
attempt += 1
delay *= 2
time.sleep(delay)
ips = self.k8s.list_pods_ips_by_field_selector(
label_selector="app=epto",
field_selector="status.phase=Running")
logger.debug("Bootstrapping", running_pods=ips, attempt=attempt)
# I populate the PartialView and I avoid to consider myself
try:
ips.remove(self.ip)
except ValueError:
logger.debug("self.ip was not there")
while not self.partialView.is_full():
random_ip = random.choice(ips)
# TODO: REPLACE WITH self.partialView.add_peer_ip(random_ip)
self.partialView.add_peer(
PodDescriptor(random_ip, random.randint(0, 9)))
logger.debug("Bootstrapping", partialView=self.partialView)
def schedule_change(self, initial_delay, interval):
initial_delay = random.randint(0, initial_delay)
time.sleep(initial_delay)
scheduler = BackgroundScheduler()
scheduler.add_job(self.shuffle_partial_view,
'interval',
seconds=interval,
max_instances=1)
scheduler.start()
def shuffle_partial_view(self):
logger.debug("Shuffling", partialView=self.partialView)
# 1) Increase by one the age of all neighbors
logger.debug("Increase by one the age of all neighbors.")
self.partialView.increment()
logger.debug("Increment", partialView=self.partialView)
# 2) Select neighbor Q with the highest age among all neighbors.
logger.debug(
"Select neighbor Q with the highest age among all neighbors.")
oldest_peer = self.partialView.get_oldest_peer()
logger.debug("SelectOldest", oldest_peer=oldest_peer)
# 3) Select l - 1 other random neighbors (meaning avoid oldest).
logger.debug(
"Select l - 1 other random neighbors (meaning avoid oldest).")
neighbors = self.partialView.select_neighbors_for_request(oldest_peer)
logger.debug("SelectNeighbors", neighbors=neighbors)
# 4) Replace Q's entry with a new entry of age 0 and with P's address.
logger.debug(
"Replace Q's entry with a new entry of age 0 and with P's address."
)
neighbors.add_peer_ip(self.ip, allow_self_ip=True)
logger.debug("AddMyself", neighbors=neighbors)
try:
# 5) Send the updated subset to peer Q.
logger.debug("Send the updated subset to peer Q (oldest_peer).",
oldest_peer=oldest_peer.ip)
response = json.loads(
self.send_message(oldest_peer.ip, 'exchange-view', neighbors))
received_partial_view = PartialView.from_dict(response.get('data'))
logger.debug("Received",
received_partial_view=received_partial_view)
# 6) I remove the oldest peer from my view
logger.debug("I remove the oldest peer from my view.",
oldest_peer=oldest_peer.ip)
self.partialView.remove_peer(oldest_peer)
logger.debug("RemovedOldest", partialView=self.partialView)
# 7) I merge my view with the one just received
logger.debug("I merge my view with the one just received.")
self.partialView.merge(neighbors, received_partial_view)
logger.debug("Merged", partialView=self.partialView)
except Timeout:
logger.error("TimeoutException: Request to " +
str(oldest_peer.ip) + " timed out.")
def send_message(self, destination_ip, path, data):
destination = os.getenv('TEST_IP',
format_address(destination_ip, 5000))
m = Message(format_address(self.ip, 5000), destination, data)
logger.debug("Request", request=m.to_json())
ret = requests.post(m.destination + '/' + self.api_version + '/' +
path,
json=m.to_json(),
timeout=5)
logger.debug("Response", response=ret.content)
return ret.content