def __init__(self, id):
self.vector_clock = VectorClock()
#state of the process, 1 means alive, 0 means dead
self.state = 1
self.message_queue = Queue()
self.kv = {}
self.count = 0
self.vector_clock.update(id, self.count)
self.id = id
self.hash_ring = ConsistentHashRing.getInstance()
self.hash_ring.add_node(self)
self.read_port = CONFIG[self.id][1]
self.write_port = CONFIG[self.id][2]
self.ip = CONFIG[self.id][0]
self.socket = socket(AF_INET, SOCK_DGRAM)
self.socket.bind((self.ip, self.read_port))
self.pending_reads = {}
self.pending_writes = {}
self.N = 3
self.R = 1
self.W = 2
self.failed_nodes = []
self.sync_kv = {}
self.check_for_sync = []
self.back = Thread(target=self.background_process)
self.history = {}
Thread.__init__(self)
self.back.start()
def perform_semantic_reconcilation(self, data):
result = -INF
vc = VectorClock()
if len(data) > 1:
clocks = []
for value, clock in data:
result = max(result, value)
clocks.append(clock)
return (result, vc.converge(clocks))
else:
return (data[0][0], data[0][1])
class CRDTStore:
def __init__(self,
server=None,
user_id=0,
initial_version=None,
initial={}):
self.state = initial
self.initial_state = initial
self.history = History()
self.user_id = user_id
self.version = VectorClock(user_id, initial_version)
self.server = server
if server is not None:
self.server.on_recieve += self.on_recieve
self.on_update = Callback()
def on_recieve(self, op: Op):
# merge
self.merge(op)
def merge(self, op: Op):
# TODO: ignore duplicates?
# update version
self.version.merge(op.id)
# apply and store history
self.state = op.apply(self.state)
self.history.append(op)
self.on_update()
def apply(self, op: Op):
# merge op
self.merge(op)
# send op to clients
if self.server is not None:
self.server.send(op)
def get(self, field: str):
if field in self.state:
return self.state[field][1] # TODO: check if tombstoned
return None
def __init__(self,
server=None,
user_id=0,
initial_version=None,
initial={}):
self.state = initial
self.initial_state = initial
self.history = History()
self.user_id = user_id
self.version = VectorClock(user_id, initial_version)
self.server = server
if server is not None:
self.server.on_recieve += self.on_recieve
self.on_update = Callback()
def erase(self):
"""Erase the global state."""
# Remove this Global State's session UUID as a valid destination.
self.service.removeDestination(self.sessionUUID)
# Reset the database.
self._dbCur.execute("DELETE FROM MessageHistory")
self._dbCon.commit()
self.clock = VectorClock()
self.clock.add(self.senderUUID)
def put(self, key, value, retries=0):
while retries < MAX_RETRIES:
dynamo_node = self.nodes[random.randint(0, len(self.nodes) - 1)]
vc = VectorClock()
vc.update(dynamo_node, 0)
value1 = (value, self.datastore.get(key, vc))
#print("Client value is "+str(value1))
put_request = Request("PUT", key, value1, None, self.port)
self.socket.settimeout(5)
Messaging.send_message(self, dynamo_node, put_request)
try:
while True:
data, addr = self.socket.recvfrom(40960)
if data:
self.socket.settimeout(None)
return pickle.loads(data)
except Exception:
self.socket.settimeout(None)
retries += 1
if config['default']['SocketType'] == 'udp':
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
elif config['default']['SocketType'] == 'tcp':
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((config['default']['ServerAddress'],
int(config['default']['ServerPort'])))
else:
print("Error: SocketType is neither udp or tcp.")
exit(1)
d_stats = dict()
d_stats['t_init'] = int(time.strftime("%s"))
vc = VectorClock(static=bool(config['VectorClock']['Static']),
max_entries=int(config['VectorClock']['MaxEntries']),
key=clientId)
log = list() # To append and recorder all sent operations.
snd = Snd(clientId,
host=config['default']['ServerAddress'],
port=int(config['default']['ServerPort']),
sock=sock,
vc=vc,
log=log,
stats=d_stats,
config=config)
rcv = Rcv(clientId,
host=config['default']['ServerAddress'],
port=int(config['default']['ServerPort']),
sock=sock,
class MessageProcessor(threading.Thread):
"""The message processor processes all incoming messages, and handles all
election and host-related messages. It also handles checking of hosts/players
leaving
"""
MOVE, CHAT, JOIN, WELCOME, LEAVE, FREEZE, UNFREEZE, NONE= range(8)
HISTORY_MESSAGE_TYPE = 'HISTORY_MESSAGE'
KEEP_ALIVE_TYPE = 'KEEP-ALIVE'
SERVER_MOVE_TYPE = 'SERVER-MESSAGE'
SERVER_ELECTED_TYPE = 'SERVER-ELECTED'
SERVER_RESPONSE_TYPE = 'SERVER-RESPONSE'
def __init__(self, service, sessionUUID, senderUUID, peerWaitingTime=30, messageWaitingTime=2):
# MulticastMessaging subclass.
if not isinstance(service, Service.OneToManyService):
raise OneToManyServiceError
self.service = service
# Identifiers.
self.sessionUUID = sessionUUID
self.senderUUID = senderUUID
self.players = None
#The last time the keep-alive message was sent (in seconds)
#self.keepAliveSentTime = 0
#The time to wait between keep-alive messages
#self.keepAliveTreshold = 1
#self.keepAliveLeftTime = 5 #the time to wait for a keep-alive message to arrive before we disconnect the player
#self.keepAliveMessages = {} # Contains the last time a keep-alive message was received per player
# Message storage.
self.inbox = Queue.Queue()
self.outbox = Queue.Queue()
# Settings.
self.peerWaitingTime = int(peerWaitingTime)
self.messageWaitingTime = int(messageWaitingTime)
# Thread state variables.
self.alive = True
self.die = False
self.lock = threading.Condition()
#Keep-alive.
self.playerRTT = {}
self.playerRTT['max'] = -1
self.lastKeepAliveSendTime = time.time()
self.sendAfterFirstKeepAlive = False
self.receivedAliveMessages = {}
# election based variables
self.host = None
# the time at which a new host was selected. If we get any new elected messages, we can ignore them
# if they happened before this time
self.hostElectedTime = 0
# list of the moves and joins we sent out, and haven't been replied to yet
# if we don't get a reply in 5 roundtrip times, the host has left, and the
# message wasn't delivered
self.messagesAwaitingApproval = []
# Other things.
self._startedWaitingForMessages = None
self.NTPoffset = 0
# Register this Global State's session UUID with the service as a
# valid destination.
self.service.registerDestination(self.sessionUUID)
super(MessageProcessor, self).__init__()
##########################################################################
# Message processing. #
##########################################################################
def sendMessage(self, message, sendToSelf = True):
"""Enqueue a message to be sent."""
with self.lock:
# print "\tGlobalState.sendMessage()", message
envelope = self._wrapMessage(message)
if sendToSelf:
self.inbox.put((self.senderUUID, message))
with self.service.lock:
# print "\tGlobalState._sendMessage()", envelope
self.service.sendMessage(self.sessionUUID, envelope)
self.service.lock.notifyAll()
def countReceivedMessages(self):
with self.lock:
return self.inbox.qsize()
def receiveMessage(self):
with self.lock:
return self.inbox.get()
def sendKeepAliveMessage(self):
with self.service.lock:
# send a keepalive message which contains the NTP time at which it was sent, so we can calculate
# Roundtrip time, and see if a player leaves the game
if self.useNTP:
self.sendMessage({'type' : self.KEEP_ALIVE_TYPE, 'originUUID':self.senderUUID, 'timestamp' : time.time() + self.NTPoffset}, False)
else:
self.sendMessage({'type' : self.KEEP_ALIVE_TYPE, 'originUUID':self.senderUUID, 'timestamp' : 0}, False)
def receiveKeepAliveMessage(self, message):
with self.lock:
uuid = message['originUUID']
#calculate the time difference between the time the message was sent and received
timeDiff = (time.time() + self.NTPoffset) - message['timestamp']
#calculate the roundtrip time
rtt = 2 * timeDiff
if self.useNTP and message['timestamp'] != 0:
self.receivedAliveMessages[uuid] = message['timestamp']
else:
rtt = 2
self.receivedAliveMessages[uuid] = time.time()
if not uuid in self.playerRTT.keys():
self.playerRTT[uuid] = rtt
else:
# calculate the average between the previous rtt for this player, and the current one
self.playerRTT[uuid] = (self.playerRTT[uuid] + rtt) / 2
max = 0
#calculate the maximum of all the roundtrip times
if not 'avg' in self.playerRTT.keys():
self.playerRTT['max'] = rtt
else:
for key in self.playerRTT.keys():
if (key != 'max' and key != 'avg'):
if self.playerRTT[key] > max:
max = self.playerRTT[key]
if max > self.playerRTT['max']:
self.playerRTT['max'] = (max + self.playerRTT['max'])/2
#calculate the average roundtrip time
if not 'avg' in self.playerRTT.keys():
self.playerRTT['avg'] = rtt
else:
avg = 0
count = 0
for key in self.playerRTT.keys():
if (key != 'max' and key != 'avg'):
avg += self.playerRTT[key]
count += 1
avg = avg / count
self.playerRTT['avg'] = avg
#checks if any players disconnected
def checkKeepAlive(self):
with self.lock:
for key in self.receivedAliveMessages.keys():
#a player has left if he hasn't send a message in 5 roundtrip times
if 5 * self.playerRTT['max'] + self.receivedAliveMessages[key] + 1 < time.time() + self.NTPoffset:
del self.receivedAliveMessages[key]
del self.playerRTT[key]
# if the player who left was the host, we have to determine the new host
if self.host != None and key == self.host:
# this player becomes the host if he has the highest UUID of all players
if len(self.receivedAliveMessages) < 1 or self.senderUUID > max (self.receivedAliveMessages.keys()):
electedMessage = {'type' : self.SERVER_ELECTED_TYPE, 'host' : self.senderUUID, 'timestamp' : time.time() + self.NTPoffset}
self.sendMessage(electedMessage)
self.receiveElectedMessage({'message':electedMessage})
self.hostLeftAt = time.time() + self.NTPoffset
self.inbox.put((key, {'type':4}))
def receiveElectedMessage(self, envelope):
if envelope['message']['timestamp'] > self.hostElectedTime:
self.hostElectedTime = envelope['message']['timestamp']
self.host = envelope['message']['host']
def receiveLeaveMessage(self, envelope):
players = copy.deepcopy(self.players)
del players[envelope['originUUID']]
if not self.useNTP and self.host != None and envelope['originUUID'] == self.host:
if len(self.players) == 1 or self.senderUUID > max (players.keys()):
electedMessage = {'type' : self.SERVER_ELECTED_TYPE, 'host' : self.senderUUID, 'timestamp' : time.time() + self.NTPoffset}
self.sendMessage(electedMessage)
self.receiveElectedMessage({'message':electedMessage})
self.inbox.put((envelope['originUUID'], {'type':4}))
# if we can't use ntp, we can't rely on the keep-alive messages
# to pick a new host, so manually select a new host
def _wrapMessage(self, message):
"""Wrap a message in an envelope to prepare it for sending."""
envelope = {}
# Add the timestamp to the envelope.
envelope['timestamp'] = time.time() + self.NTPoffset
# Add the sender UUUID and the original sender UUID to the envelope
# (which is always ourselves).
envelope['senderUUID'] = envelope['originUUID'] = self.senderUUID
# Store the message in the envelope.
envelope['message'] = message
return envelope
def messageApproval(self, message):
""" Approves a message sent by this user """
if message['target'] == self.senderUUID:
# If we received a history message, it means we have correctly joined the game
if message['type'] == self.HISTORY_MESSAGE_TYPE:
for m in self.messagesAwaitingApproval:
# Remove the join message
if m['type'] == self.JOIN:
self.messagesAwaitingApproval.remove(m)
# If we received a server response message, it means the move we sent was processed
if message['type'] == self.SERVER_RESPONSE_TYPE:
for m in self.messagesAwaitingApproval:
# Delete the move message
if m['type'] == self.SERVER_MOVE_TYPE and m['col'] == message['col']:
self.messagesAwaitingApproval.remove(m)
def checkApproval(self):
""" Check if a message sent by this user was approved and sends it again if it timed out """
for m in self.messagesAwaitingApproval:
# wait till a new host was elected before sending the message again
if m['timestamp'] < self.hostElectedTime:
# a message times out if it wasn't approved after 5 roundtrip times
if 5 * self.playerRTT['max'] + m['timestamp'] + 1 < time.time() + self.NTPoffset:
# send the message again
self.sendMessage(m)
self.messagesAwaitingApproval.remove(m)
if self.senderUUID != self.host:
m['timestamp'] = time.time() + self.NTPoffset
self.messagesAwaitingApproval.append(m)
def processMessage(self, envelope):
"""Process a message and put it in the inbox if its meant for us"""
# If the message was a request to make a move, do the move, and let the player who
# did the move know his message was processed
if envelope['message']['type'] == self.SERVER_MOVE_TYPE:
if envelope['message']['target'] == self.senderUUID:
with self.lock:
self.sendMessage({'type' : self.SERVER_RESPONSE_TYPE, 'col' : envelope['message']['col'], 'target' : envelope['originUUID']})
self.inbox.put((envelope['originUUID'], envelope['message']))
# If the message was an approval of a message sent by this user, approve that message
elif envelope['message']['type'] == self.SERVER_RESPONSE_TYPE:
self.messageApproval(envelope['message'])
# Process a keep-alive message
elif envelope['message']['type'] == self.KEEP_ALIVE_TYPE:
self.receiveKeepAliveMessage(envelope)
# Process an elected message
elif envelope['message']['type'] == self.SERVER_ELECTED_TYPE:
self.receiveElectedMessage(envelope)
#if the message is a LEAVE message
elif envelope['message']['type'] == 4:
self.receiveLeaveMessage(envelope)
else:
# Move the message to the inbox queue, so it can be retrieved.
with self.lock:
self.inbox.put((envelope['originUUID'], envelope['message']))
def erase(self):
"""Erase the global state."""
# Remove this Global State's session UUID as a valid destination.
self.service.removeDestination(self.sessionUUID)
# Reset the database.
self._dbCur.execute("DELETE FROM MessageHistory")
self._dbCon.commit()
self.clock = VectorClock()
self.clock.add(self.senderUUID)
def getNTPoffset(self):
# get the NTP offset of this computers clock
try:
client = ntplib.NTPClient()
response = client.request('europe.pool.ntp.org', version=3)
if response.leap != 3:
self.NTPoffset = response.offset
self.useNTP = True
except:
print 'Warning! NTP server could not be reached'
self.useNTP = False
pass
# print 'NTPoffset is now: ' + str(self.NTPoffset)
def run(self):
self.getNTPoffset()
while self.alive:
# Check if it's time to send liveness messages.
# TODO
# Retrieve incoming messages and store them in the waiting room.
with self.lock:
with self.service.lock:
if self.service.countReceivedMessages(self.sessionUUID) > 0:
envelope = self.service.receiveMessage(self.sessionUUID)
# Ignore our own messages.
if envelope['senderUUID'] != self.senderUUID:
self.processMessage(envelope)
# If there are other players, send keepalive messages with an interval suitable
# to their roundtrip time
if self.useNTP and ('avg' in self.playerRTT.keys()):
if(float(min(self.playerRTT['avg'], 1)) < float(time.time() - self.lastKeepAliveSendTime)):
self.sendKeepAliveMessage()
self.checkKeepAlive()
# check if move and join requests were received
self.checkApproval()
self.lastKeepAliveSendTime = time.time()
elif time.time() - self.lastKeepAliveSendTime > 1:
self.keepAliveSent = True
self.sendKeepAliveMessage()
self.checkKeepAlive()
self.checkApproval()
self.lastKeepAliveSendTime = time.time()
# 100 refreshes per second is plenty.
time.sleep(0.01)
def kill(self):
# Let the thread know it should commit suicide. But first let it send
# all remaining messages.
# while self.outbox.qsize() > 0:
# if
with self.lock:
self._commitSuicide()
def _commitSuicide(self):
"""Commit suicide when asked to. The lock must be acquired before
calling this method.
"""
# Stop us from running any further.
self.alive = False
class Node(Thread):
def __init__(self, id):
self.vector_clock = VectorClock()
#state of the process, 1 means alive, 0 means dead
self.state = 1
self.message_queue = Queue()
self.kv = {}
self.count = 0
self.vector_clock.update(id, self.count)
self.id = id
self.hash_ring = ConsistentHashRing.getInstance()
self.hash_ring.add_node(self)
self.read_port = CONFIG[self.id][1]
self.write_port = CONFIG[self.id][2]
self.ip = CONFIG[self.id][0]
self.socket = socket(AF_INET, SOCK_DGRAM)
self.socket.bind((self.ip, self.read_port))
self.pending_reads = {}
self.pending_writes = {}
self.N = 3
self.R = 1
self.W = 2
self.failed_nodes = []
self.sync_kv = {}
self.check_for_sync = []
self.back = Thread(target=self.background_process)
self.history = {}
Thread.__init__(self)
self.back.start()
def get_sequence_no(self):
self.count += 1
return self.count
def perform_antientropy(self):
for key in self.kv:
preference_list = self.hash_ring.get_node(key)
for node in preference_list:
sync_message = Request("ENTROPY", key, self.kv[key])
Messaging.send_message(self, node.id, sync_message)
def antientropy(self, *data):
dict = data[0]
if dict.key not in self.kv:
self.kv[dict.key] = dict.value
else:
list_other = self.kv[dict.key]
list_other += dict.value
self.kv[dict.key] = self.perform_syntactic_reconcilation(
list_other)
def handoff(self):
for node in self.check_for_sync:
keys = list(self.sync_kv[node].keys())
values = list(self.sync_kv[node].values())
synchronize = Request("SYNC", keys, values,
generate_random_number())
Messaging.send_message(self, node, synchronize)
def synchronize(self, msg):
keys = msg.key
values = msg.value
for i in range(len(keys)):
if keys[i] not in self.kv:
self.kv[keys[i]] = []
self.kv[keys[i]] += values[i]
for key in keys:
self.kv[key] = self.perform_syntactic_reconcilation(self.kv[key])
#print(self.id+" "+str(self.kv))
def checkIfAlive(self):
for node in self.failed_nodes:
ping = Request("PING", None)
Messaging.send_message(self, node, ping)
def background_process(self):
try:
while True:
if self.state == 1:
self.handoff()
self.checkIfAlive()
#self.perform_antientropy()
time.sleep(1)
except Exception:
self.background_process()
def perform_put(self, *data):
dict = data[0]
key = dict.key
preference_list = self.hash_ring.get_node(key, self.failed_nodes)
if (self not in preference_list):
coordinator = preference_list[0].id
dict = Request("FORWARD-PUT", dict.key, dict.value,
generate_random_number(), dict.client)
Messaging.send_message(self, coordinator, dict)
time.sleep(3)
if REQUESTS.get(dict.request, False) != True:
#print("Timedout PUT")
dict.action = "PUT"
dict.request = generate_random_number()
self.socket.settimeout(None)
self.failed_nodes.append(coordinator)
self.perform_put(dict)
else:
self.vector_clock.update(self.id, self.get_sequence_no())
metadata = deepcopy(self.vector_clock)
if dict.value[1] > metadata:
metadata = dict.value[1]
dict.value = (dict.value[0], metadata)
dict.request = generate_random_number()
Messaging.broadcast_put(self, preference_list, dict)
def perform_syntactic_reconcilation(self, list):
dict = {}
clocks = []
#print("list is "+str(list)+" "+self.id)
for value, clock in list:
dict[clock] = value
clocks.append(clock)
result = self.vector_clock.combine(clocks)
final_result = []
for clock in result:
final_result.append((dict[clock], clock))
return final_result
def perform_store(self, data):
time.sleep(random.randint(0, 1000) / 1000)
dict = data[0]
addr = data[1]
self.vector_clock.update(self.id, self.get_sequence_no())
self.vector_clock = self.vector_clock.converge(
[self.vector_clock, dict.value[1]])
if dict.key not in self.kv:
self.kv[dict.key] = list()
self.kv[dict.key].append(dict.value)
self.kv[dict.key] = self.perform_syntactic_reconcilation(
self.kv[dict.key])
dict = Request("ACK-PUT", None, None, dict.request)
Messaging.send_message(self, PORT_TO_ID[addr[1]], dict)
def perform_get(self, dict):
key = dict.key
preference_list = self.hash_ring.get_node(key, self.failed_nodes)
if (self not in preference_list):
coordinator = preference_list[0].id
dict = Request("FORWARD-GET", dict.key, dict.value,
generate_random_number(), dict.client)
Messaging.send_message(self, coordinator, dict)
time.sleep(3)
if REQUESTS.get(dict.request, False) != True:
#print("Timedout GET")
dict.action = "GET"
dict.request = generate_random_number()
self.failed_nodes.append(coordinator)
self.perform_get(dict)
else:
dict.request = generate_random_number()
Messaging.broadcast_get(self, preference_list, dict)
def retreive_key(self, *data):
time.sleep(random.randint(0, 1000) / 1000)
dict = data[0]
from_node = data[1]
val = self.kv.get(dict.key, None)
response = Request("ACK-GET", dict.key, val, dict.request)
Messaging.send_message(self, from_node, response)
#Utility #Print function
def string(self, dict):
temp = "{"
for key in dict:
temp = temp + "," + str(key) + ":"
temp2 = "["
for val in dict[key]:
temp2 += str(val[0]) + "," + str(val[1].clock) + ","
temp2 += "]"
temp += temp2
temp += "}"
return temp
# Message gets uncompressed first
def run(self):
try:
while True:
data, addr = self.socket.recvfrom(MAX_MESSAGE_SIZE)
dict = pickle.loads(data)
# #print(self.id + " received data from" + str(addr[1]))
if self.state == 1:
if dict.action == "PUT":
#print(self.id+" received PUT "+str(dict.key)+" "+str(dict.value)+" "+str(dict.client))
thread1 = Thread(target=self.perform_put,
args=(dict, ))
thread1.start()
if dict.action == "STORE":
#print(self.id + " received STORE"+str(dict.key)+" "+str(dict.value[0]))
thread2 = Thread(target=self.perform_store,
args=([dict, addr], ))
thread2.start()
#print(self.id+" "+self.string(self.kv))
if dict.action == "GET":
#print(self.id+" Received Get")
thread3 = Thread(target=self.perform_get,
args=(dict, ))
thread3.start()
if dict.action == "FETCH":
#print(self.id+" Received Fetch")
thread4 = Thread(target=self.retreive_key,
args=[dict, PORT_TO_ID[addr[1]]])
thread4.start()
if dict.action == "EXIT":
#print(self.id+" Will Fail now")
self.state = 0
if dict.action == "FORWARD-PUT":
#print(self.id+"Received Forward")
dict.action = "PUT"
response = Request("ACK-FORWARD-PUT", None, None,
dict.request)
self.socket.sendto(pickle.dumps(response), addr)
thread5 = Thread(target=self.perform_put,
args=(dict, ))
thread5.start()
#self.perform_put(dict)
if dict.action == "FORWARD-GET":
#print(self.id+"Received Forward")
dict.action = "GET"
response = Request("ACK-FORWARD-GET", None, None,
dict.request)
self.socket.sendto(pickle.dumps(response), addr)
threada = Thread(target=self.perform_get, args=[dict])
threada.start()
if dict.action == "PING":
#print(self.id+"Received Ping")
response = Request("PONG", None)
Messaging.send_message(self, PORT_TO_ID[addr[1]],
response)
if dict.action == "ACK-FORWARD-PUT":
#print("Received forward PUT ack")
REQUESTS[dict.request] = True
if dict.action == "ACK-FORWARD-GET":
#print("Received forward Get ack")
REQUESTS[dict.request] = True
if dict.action == "ACK-PUT":
#print("PUT ACK received by "+self.id +" from "+ PORT_TO_ID[addr[1]])
if dict.request not in HISTORY:
HISTORY[dict.request] = set()
HISTORY[dict.request].add(PORT_TO_ID[addr[1]])
if dict.action == "ACK-GET":
#print("GET ACK received by "+self.id +" from "+ PORT_TO_ID[addr[1]])
if dict.request not in HISTORY:
HISTORY[dict.request] = list()
HISTORY[dict.request].append(
(PORT_TO_ID[addr[1]], dict.value))
if dict.action == "SYNC":
#print(self.id+"Received handoff Sync")
thread10 = Thread(target=self.synchronize, args=[dict])
thread10.start()
response = Request("SYNC-ACK", None)
Messaging.send_message(self, PORT_TO_ID[addr[1]],
response)
if (dict.action == "SYNC-ACK"):
#print(self.id + " Synced " + PORT_TO_ID[addr[1]])
self.sync_kv.pop(PORT_TO_ID[addr[1]], None)
self.check_for_sync.remove(PORT_TO_ID[addr[1]])
if (dict.action == "PONG"):
self.failed_nodes.remove(PORT_TO_ID[addr[1]])
#print("node" + str(PORT_TO_ID[addr[1]])+" is alive")
#print("Failed nodes=" + str(self.failed_nodes))
if (dict.action == "ENTROPY"):
##print(self.id+" received entropy")
threadb = Thread(target=self.antientropy, args=[dict])
threadb.start()
else:
if dict.action == "REVIVE":
#print(self.id+" is reviving")
self.state = 1
except Exception:
self.run()
def checker(self):
# If item is ready to be delivered go ahead.
# And then check if the limbo item can be delivered.
# Else add it to the limbo queue
# And ask for the missing elements
# Lets go!
while True:
try:
item = json.loads(self.q_rcv.get())
except ValueError:
logging.debug("JSON Couldn't load the message!")
continue
else:
#print(item)
msg_type = item['type']
# reassemble the vc if needed
if msg_type == 'DATA_2ALL' or msg_type == 'DATA_P2P' or msg_type == 'recovery':
aux = VectorClock(static=True,
max_entries=item['vc']['max_entries'],
key=item['vc']['key'])
aux.vc = item['vc']['vc']
item['vc'] = aux
self.d_stats['last_vc_seen'] = item['vc']
if msg_type == 'recovery':
self.d_stats['count_recovery'] += 1
self.d_stats['count_received'] += 1
logging.debug("[stats] (R, " + str(item['src_id']) + ", " +
str(self.src_id) + ", " +
str(item['vc'].vc[item['src_id']]) + ")")
# Lets check if message is in the rer_worker waiting list
# if its there we remove it and continue as normal.
if self.inRER(item):
del self.rer[(item['vc'].vc[item['src_id']],
item['src_id'])]
self.d_stats['tvop'] += 1
# if the message is duplicated, means that it should already be delivered or in limbo
if not self.duplicate(item['vc']):
# Its ugly but we can check it here. for , A1, A2, A3 cases.
if msg_type == 'recovery':
logging.debug(
"I'm in receovery!( I know... its not funny)")
if self.vc.vc[item['src_id']] >= item['vc'].vc[
item['src_id']]:
self.d_stats['count_fp'] += 1
if self.vc.happened_before(item['vc']):
# item can be delivered.
self.vc.inc(item['vc'].key)
self.q_deliv.put(item['msg'])
#logging.debug("[stats] (D, " + str(item['src_id']) + ", " + str(self.src_id) + ", " + str(item['vc'].vc[item['src_id']]) + ")")
if msg_type == 'recovery':
# the message has been recovered. with the recovery
logging.debug(
"[stats] (DR, " + str(item['src_id']) +
", " + str(self.src_id) + ", " +
str(item['vc'].vc[item['src_id']]) + ")")
else:
logging.debug(
"[stats] (D, " + str(item['src_id']) +
", " + str(self.src_id) + ", " +
str(item['vc'].vc[item['src_id']]) + ")")
# Now Check if limbo stuff can be delivered.
mark_for_deletion = list()
len_i = len(self.l_limbo)
for i in range(len_i):
item = self.l_limbo[i]
if self.vc.happened_before(item['vc']):
self.vc.inc(item['vc'].key)
self.q_deliv.put(item['msg'])
logging.debug(
"[stats] (D, " + str(item['src_id']) +
", " + str(self.src_id) + ", " +
str(item['vc'].vc[item['src_id']]) +
")")
# its been delivered, so leave the limbo
#del self.l_limbo[i]
mark_for_deletion.append(i)
else:
# ask again
self.check_missing(item)
for i in reversed(mark_for_deletion):
del self.l_limbo[i]
else:
#it couldnt be delivered, so add to limbo and ask for missing msgs
self.l_limbo.append(item)
self.d_stats["count_limbo"] += 1
self.check_missing(item)
else:
if msg_type == 'recovery':
self.d_stats['count_fp'] += 1
elif msg_type == 'DATA_2ALL' or msg_type == 'DATA_P2P':
self.d_stats['count_A3'] += 1
# It was a duplicate but... you can check if something can be delivered.
# Check in the limbo
self.check_limbo()
elif msg_type == 'ack':
# What do we want to do with ack??
self.d_stats['count_ack'] += 1
elif msg_type == 'missing':
# We send what they want.
missing_msg = self.log[item['num']]
self.snd.recovery(dst_id=item['src_id'],
str_msg=missing_msg['msg'],
vc=missing_msg['vc'])
self.d_stats['count_missing'] += 1
elif msg_type == 'nack':
# Something went wrong with the AR.
self.d_stats['count_nack'] += 1
elif msg_type == 'debug':
print(item['msg'])
elif msg_type == 'ack_bj':
# We update the membership dict with what was sent.
self.d_stats['membership'] = item['msg']
# If dst_id exists but the address has changed, then we are screwed.
for dst_id, address in self.d_stats['membership'].items():
if not dst_id == self.src_id:
if not dst_id in self.d_stats['ms_conns']:
# We have to create the connection to this address.
sock = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
sock.setsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY, 1)
sock.connect((address,
int(self.d_stats['config']
['default']['ServerPort'])))
self.d_stats['ms_conns'][dst_id] = sock
v_fileno = list()
v_id = list()
for dst_id, conn in self.d_stats['ms_conns'].items():
v_fileno.append(conn.fileno())
v_id.append(dst_id)
self.d_stats['ms_fileno'] = v_fileno
self.d_stats['ms_fileno_id'] = v_id
self.q_rcv.task_done()
