def __init__(self, id): self.vector_clock = VectorClock() #state of the process, 1 means alive, 0 means dead self.state = 1 self.message_queue = Queue() self.kv = {} self.count = 0 self.vector_clock.update(id, self.count) self.id = id self.hash_ring = ConsistentHashRing.getInstance() self.hash_ring.add_node(self) self.read_port = CONFIG[self.id][1] self.write_port = CONFIG[self.id][2] self.ip = CONFIG[self.id][0] self.socket = socket(AF_INET, SOCK_DGRAM) self.socket.bind((self.ip, self.read_port)) self.pending_reads = {} self.pending_writes = {} self.N = 3 self.R = 1 self.W = 2 self.failed_nodes = [] self.sync_kv = {} self.check_for_sync = [] self.back = Thread(target=self.background_process) self.history = {} Thread.__init__(self) self.back.start()
def perform_semantic_reconcilation(self, data): result = -INF vc = VectorClock() if len(data) > 1: clocks = [] for value, clock in data: result = max(result, value) clocks.append(clock) return (result, vc.converge(clocks)) else: return (data[0][0], data[0][1])
class CRDTStore: def __init__(self, server=None, user_id=0, initial_version=None, initial={}): self.state = initial self.initial_state = initial self.history = History() self.user_id = user_id self.version = VectorClock(user_id, initial_version) self.server = server if server is not None: self.server.on_recieve += self.on_recieve self.on_update = Callback() def on_recieve(self, op: Op): # merge self.merge(op) def merge(self, op: Op): # TODO: ignore duplicates? # update version self.version.merge(op.id) # apply and store history self.state = op.apply(self.state) self.history.append(op) self.on_update() def apply(self, op: Op): # merge op self.merge(op) # send op to clients if self.server is not None: self.server.send(op) def get(self, field: str): if field in self.state: return self.state[field][1] # TODO: check if tombstoned return None
def __init__(self, server=None, user_id=0, initial_version=None, initial={}): self.state = initial self.initial_state = initial self.history = History() self.user_id = user_id self.version = VectorClock(user_id, initial_version) self.server = server if server is not None: self.server.on_recieve += self.on_recieve self.on_update = Callback()
def erase(self): """Erase the global state.""" # Remove this Global State's session UUID as a valid destination. self.service.removeDestination(self.sessionUUID) # Reset the database. self._dbCur.execute("DELETE FROM MessageHistory") self._dbCon.commit() self.clock = VectorClock() self.clock.add(self.senderUUID)
def put(self, key, value, retries=0): while retries < MAX_RETRIES: dynamo_node = self.nodes[random.randint(0, len(self.nodes) - 1)] vc = VectorClock() vc.update(dynamo_node, 0) value1 = (value, self.datastore.get(key, vc)) #print("Client value is "+str(value1)) put_request = Request("PUT", key, value1, None, self.port) self.socket.settimeout(5) Messaging.send_message(self, dynamo_node, put_request) try: while True: data, addr = self.socket.recvfrom(40960) if data: self.socket.settimeout(None) return pickle.loads(data) except Exception: self.socket.settimeout(None) retries += 1
if config['default']['SocketType'] == 'udp': sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) elif config['default']['SocketType'] == 'tcp': sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((config['default']['ServerAddress'], int(config['default']['ServerPort']))) else: print("Error: SocketType is neither udp or tcp.") exit(1) d_stats = dict() d_stats['t_init'] = int(time.strftime("%s")) vc = VectorClock(static=bool(config['VectorClock']['Static']), max_entries=int(config['VectorClock']['MaxEntries']), key=clientId) log = list() # To append and recorder all sent operations. snd = Snd(clientId, host=config['default']['ServerAddress'], port=int(config['default']['ServerPort']), sock=sock, vc=vc, log=log, stats=d_stats, config=config) rcv = Rcv(clientId, host=config['default']['ServerAddress'], port=int(config['default']['ServerPort']), sock=sock,
class MessageProcessor(threading.Thread): """The message processor processes all incoming messages, and handles all election and host-related messages. It also handles checking of hosts/players leaving """ MOVE, CHAT, JOIN, WELCOME, LEAVE, FREEZE, UNFREEZE, NONE= range(8) HISTORY_MESSAGE_TYPE = 'HISTORY_MESSAGE' KEEP_ALIVE_TYPE = 'KEEP-ALIVE' SERVER_MOVE_TYPE = 'SERVER-MESSAGE' SERVER_ELECTED_TYPE = 'SERVER-ELECTED' SERVER_RESPONSE_TYPE = 'SERVER-RESPONSE' def __init__(self, service, sessionUUID, senderUUID, peerWaitingTime=30, messageWaitingTime=2): # MulticastMessaging subclass. if not isinstance(service, Service.OneToManyService): raise OneToManyServiceError self.service = service # Identifiers. self.sessionUUID = sessionUUID self.senderUUID = senderUUID self.players = None #The last time the keep-alive message was sent (in seconds) #self.keepAliveSentTime = 0 #The time to wait between keep-alive messages #self.keepAliveTreshold = 1 #self.keepAliveLeftTime = 5 #the time to wait for a keep-alive message to arrive before we disconnect the player #self.keepAliveMessages = {} # Contains the last time a keep-alive message was received per player # Message storage. self.inbox = Queue.Queue() self.outbox = Queue.Queue() # Settings. self.peerWaitingTime = int(peerWaitingTime) self.messageWaitingTime = int(messageWaitingTime) # Thread state variables. self.alive = True self.die = False self.lock = threading.Condition() #Keep-alive. self.playerRTT = {} self.playerRTT['max'] = -1 self.lastKeepAliveSendTime = time.time() self.sendAfterFirstKeepAlive = False self.receivedAliveMessages = {} # election based variables self.host = None # the time at which a new host was selected. If we get any new elected messages, we can ignore them # if they happened before this time self.hostElectedTime = 0 # list of the moves and joins we sent out, and haven't been replied to yet # if we don't get a reply in 5 roundtrip times, the host has left, and the # message wasn't delivered self.messagesAwaitingApproval = [] # Other things. self._startedWaitingForMessages = None self.NTPoffset = 0 # Register this Global State's session UUID with the service as a # valid destination. self.service.registerDestination(self.sessionUUID) super(MessageProcessor, self).__init__() ########################################################################## # Message processing. # ########################################################################## def sendMessage(self, message, sendToSelf = True): """Enqueue a message to be sent.""" with self.lock: # print "\tGlobalState.sendMessage()", message envelope = self._wrapMessage(message) if sendToSelf: self.inbox.put((self.senderUUID, message)) with self.service.lock: # print "\tGlobalState._sendMessage()", envelope self.service.sendMessage(self.sessionUUID, envelope) self.service.lock.notifyAll() def countReceivedMessages(self): with self.lock: return self.inbox.qsize() def receiveMessage(self): with self.lock: return self.inbox.get() def sendKeepAliveMessage(self): with self.service.lock: # send a keepalive message which contains the NTP time at which it was sent, so we can calculate # Roundtrip time, and see if a player leaves the game if self.useNTP: self.sendMessage({'type' : self.KEEP_ALIVE_TYPE, 'originUUID':self.senderUUID, 'timestamp' : time.time() + self.NTPoffset}, False) else: self.sendMessage({'type' : self.KEEP_ALIVE_TYPE, 'originUUID':self.senderUUID, 'timestamp' : 0}, False) def receiveKeepAliveMessage(self, message): with self.lock: uuid = message['originUUID'] #calculate the time difference between the time the message was sent and received timeDiff = (time.time() + self.NTPoffset) - message['timestamp'] #calculate the roundtrip time rtt = 2 * timeDiff if self.useNTP and message['timestamp'] != 0: self.receivedAliveMessages[uuid] = message['timestamp'] else: rtt = 2 self.receivedAliveMessages[uuid] = time.time() if not uuid in self.playerRTT.keys(): self.playerRTT[uuid] = rtt else: # calculate the average between the previous rtt for this player, and the current one self.playerRTT[uuid] = (self.playerRTT[uuid] + rtt) / 2 max = 0 #calculate the maximum of all the roundtrip times if not 'avg' in self.playerRTT.keys(): self.playerRTT['max'] = rtt else: for key in self.playerRTT.keys(): if (key != 'max' and key != 'avg'): if self.playerRTT[key] > max: max = self.playerRTT[key] if max > self.playerRTT['max']: self.playerRTT['max'] = (max + self.playerRTT['max'])/2 #calculate the average roundtrip time if not 'avg' in self.playerRTT.keys(): self.playerRTT['avg'] = rtt else: avg = 0 count = 0 for key in self.playerRTT.keys(): if (key != 'max' and key != 'avg'): avg += self.playerRTT[key] count += 1 avg = avg / count self.playerRTT['avg'] = avg #checks if any players disconnected def checkKeepAlive(self): with self.lock: for key in self.receivedAliveMessages.keys(): #a player has left if he hasn't send a message in 5 roundtrip times if 5 * self.playerRTT['max'] + self.receivedAliveMessages[key] + 1 < time.time() + self.NTPoffset: del self.receivedAliveMessages[key] del self.playerRTT[key] # if the player who left was the host, we have to determine the new host if self.host != None and key == self.host: # this player becomes the host if he has the highest UUID of all players if len(self.receivedAliveMessages) < 1 or self.senderUUID > max (self.receivedAliveMessages.keys()): electedMessage = {'type' : self.SERVER_ELECTED_TYPE, 'host' : self.senderUUID, 'timestamp' : time.time() + self.NTPoffset} self.sendMessage(electedMessage) self.receiveElectedMessage({'message':electedMessage}) self.hostLeftAt = time.time() + self.NTPoffset self.inbox.put((key, {'type':4})) def receiveElectedMessage(self, envelope): if envelope['message']['timestamp'] > self.hostElectedTime: self.hostElectedTime = envelope['message']['timestamp'] self.host = envelope['message']['host'] def receiveLeaveMessage(self, envelope): players = copy.deepcopy(self.players) del players[envelope['originUUID']] if not self.useNTP and self.host != None and envelope['originUUID'] == self.host: if len(self.players) == 1 or self.senderUUID > max (players.keys()): electedMessage = {'type' : self.SERVER_ELECTED_TYPE, 'host' : self.senderUUID, 'timestamp' : time.time() + self.NTPoffset} self.sendMessage(electedMessage) self.receiveElectedMessage({'message':electedMessage}) self.inbox.put((envelope['originUUID'], {'type':4})) # if we can't use ntp, we can't rely on the keep-alive messages # to pick a new host, so manually select a new host def _wrapMessage(self, message): """Wrap a message in an envelope to prepare it for sending.""" envelope = {} # Add the timestamp to the envelope. envelope['timestamp'] = time.time() + self.NTPoffset # Add the sender UUUID and the original sender UUID to the envelope # (which is always ourselves). envelope['senderUUID'] = envelope['originUUID'] = self.senderUUID # Store the message in the envelope. envelope['message'] = message return envelope def messageApproval(self, message): """ Approves a message sent by this user """ if message['target'] == self.senderUUID: # If we received a history message, it means we have correctly joined the game if message['type'] == self.HISTORY_MESSAGE_TYPE: for m in self.messagesAwaitingApproval: # Remove the join message if m['type'] == self.JOIN: self.messagesAwaitingApproval.remove(m) # If we received a server response message, it means the move we sent was processed if message['type'] == self.SERVER_RESPONSE_TYPE: for m in self.messagesAwaitingApproval: # Delete the move message if m['type'] == self.SERVER_MOVE_TYPE and m['col'] == message['col']: self.messagesAwaitingApproval.remove(m) def checkApproval(self): """ Check if a message sent by this user was approved and sends it again if it timed out """ for m in self.messagesAwaitingApproval: # wait till a new host was elected before sending the message again if m['timestamp'] < self.hostElectedTime: # a message times out if it wasn't approved after 5 roundtrip times if 5 * self.playerRTT['max'] + m['timestamp'] + 1 < time.time() + self.NTPoffset: # send the message again self.sendMessage(m) self.messagesAwaitingApproval.remove(m) if self.senderUUID != self.host: m['timestamp'] = time.time() + self.NTPoffset self.messagesAwaitingApproval.append(m) def processMessage(self, envelope): """Process a message and put it in the inbox if its meant for us""" # If the message was a request to make a move, do the move, and let the player who # did the move know his message was processed if envelope['message']['type'] == self.SERVER_MOVE_TYPE: if envelope['message']['target'] == self.senderUUID: with self.lock: self.sendMessage({'type' : self.SERVER_RESPONSE_TYPE, 'col' : envelope['message']['col'], 'target' : envelope['originUUID']}) self.inbox.put((envelope['originUUID'], envelope['message'])) # If the message was an approval of a message sent by this user, approve that message elif envelope['message']['type'] == self.SERVER_RESPONSE_TYPE: self.messageApproval(envelope['message']) # Process a keep-alive message elif envelope['message']['type'] == self.KEEP_ALIVE_TYPE: self.receiveKeepAliveMessage(envelope) # Process an elected message elif envelope['message']['type'] == self.SERVER_ELECTED_TYPE: self.receiveElectedMessage(envelope) #if the message is a LEAVE message elif envelope['message']['type'] == 4: self.receiveLeaveMessage(envelope) else: # Move the message to the inbox queue, so it can be retrieved. with self.lock: self.inbox.put((envelope['originUUID'], envelope['message'])) def erase(self): """Erase the global state.""" # Remove this Global State's session UUID as a valid destination. self.service.removeDestination(self.sessionUUID) # Reset the database. self._dbCur.execute("DELETE FROM MessageHistory") self._dbCon.commit() self.clock = VectorClock() self.clock.add(self.senderUUID) def getNTPoffset(self): # get the NTP offset of this computers clock try: client = ntplib.NTPClient() response = client.request('europe.pool.ntp.org', version=3) if response.leap != 3: self.NTPoffset = response.offset self.useNTP = True except: print 'Warning! NTP server could not be reached' self.useNTP = False pass # print 'NTPoffset is now: ' + str(self.NTPoffset) def run(self): self.getNTPoffset() while self.alive: # Check if it's time to send liveness messages. # TODO # Retrieve incoming messages and store them in the waiting room. with self.lock: with self.service.lock: if self.service.countReceivedMessages(self.sessionUUID) > 0: envelope = self.service.receiveMessage(self.sessionUUID) # Ignore our own messages. if envelope['senderUUID'] != self.senderUUID: self.processMessage(envelope) # If there are other players, send keepalive messages with an interval suitable # to their roundtrip time if self.useNTP and ('avg' in self.playerRTT.keys()): if(float(min(self.playerRTT['avg'], 1)) < float(time.time() - self.lastKeepAliveSendTime)): self.sendKeepAliveMessage() self.checkKeepAlive() # check if move and join requests were received self.checkApproval() self.lastKeepAliveSendTime = time.time() elif time.time() - self.lastKeepAliveSendTime > 1: self.keepAliveSent = True self.sendKeepAliveMessage() self.checkKeepAlive() self.checkApproval() self.lastKeepAliveSendTime = time.time() # 100 refreshes per second is plenty. time.sleep(0.01) def kill(self): # Let the thread know it should commit suicide. But first let it send # all remaining messages. # while self.outbox.qsize() > 0: # if with self.lock: self._commitSuicide() def _commitSuicide(self): """Commit suicide when asked to. The lock must be acquired before calling this method. """ # Stop us from running any further. self.alive = False
class Node(Thread): def __init__(self, id): self.vector_clock = VectorClock() #state of the process, 1 means alive, 0 means dead self.state = 1 self.message_queue = Queue() self.kv = {} self.count = 0 self.vector_clock.update(id, self.count) self.id = id self.hash_ring = ConsistentHashRing.getInstance() self.hash_ring.add_node(self) self.read_port = CONFIG[self.id][1] self.write_port = CONFIG[self.id][2] self.ip = CONFIG[self.id][0] self.socket = socket(AF_INET, SOCK_DGRAM) self.socket.bind((self.ip, self.read_port)) self.pending_reads = {} self.pending_writes = {} self.N = 3 self.R = 1 self.W = 2 self.failed_nodes = [] self.sync_kv = {} self.check_for_sync = [] self.back = Thread(target=self.background_process) self.history = {} Thread.__init__(self) self.back.start() def get_sequence_no(self): self.count += 1 return self.count def perform_antientropy(self): for key in self.kv: preference_list = self.hash_ring.get_node(key) for node in preference_list: sync_message = Request("ENTROPY", key, self.kv[key]) Messaging.send_message(self, node.id, sync_message) def antientropy(self, *data): dict = data[0] if dict.key not in self.kv: self.kv[dict.key] = dict.value else: list_other = self.kv[dict.key] list_other += dict.value self.kv[dict.key] = self.perform_syntactic_reconcilation( list_other) def handoff(self): for node in self.check_for_sync: keys = list(self.sync_kv[node].keys()) values = list(self.sync_kv[node].values()) synchronize = Request("SYNC", keys, values, generate_random_number()) Messaging.send_message(self, node, synchronize) def synchronize(self, msg): keys = msg.key values = msg.value for i in range(len(keys)): if keys[i] not in self.kv: self.kv[keys[i]] = [] self.kv[keys[i]] += values[i] for key in keys: self.kv[key] = self.perform_syntactic_reconcilation(self.kv[key]) #print(self.id+" "+str(self.kv)) def checkIfAlive(self): for node in self.failed_nodes: ping = Request("PING", None) Messaging.send_message(self, node, ping) def background_process(self): try: while True: if self.state == 1: self.handoff() self.checkIfAlive() #self.perform_antientropy() time.sleep(1) except Exception: self.background_process() def perform_put(self, *data): dict = data[0] key = dict.key preference_list = self.hash_ring.get_node(key, self.failed_nodes) if (self not in preference_list): coordinator = preference_list[0].id dict = Request("FORWARD-PUT", dict.key, dict.value, generate_random_number(), dict.client) Messaging.send_message(self, coordinator, dict) time.sleep(3) if REQUESTS.get(dict.request, False) != True: #print("Timedout PUT") dict.action = "PUT" dict.request = generate_random_number() self.socket.settimeout(None) self.failed_nodes.append(coordinator) self.perform_put(dict) else: self.vector_clock.update(self.id, self.get_sequence_no()) metadata = deepcopy(self.vector_clock) if dict.value[1] > metadata: metadata = dict.value[1] dict.value = (dict.value[0], metadata) dict.request = generate_random_number() Messaging.broadcast_put(self, preference_list, dict) def perform_syntactic_reconcilation(self, list): dict = {} clocks = [] #print("list is "+str(list)+" "+self.id) for value, clock in list: dict[clock] = value clocks.append(clock) result = self.vector_clock.combine(clocks) final_result = [] for clock in result: final_result.append((dict[clock], clock)) return final_result def perform_store(self, data): time.sleep(random.randint(0, 1000) / 1000) dict = data[0] addr = data[1] self.vector_clock.update(self.id, self.get_sequence_no()) self.vector_clock = self.vector_clock.converge( [self.vector_clock, dict.value[1]]) if dict.key not in self.kv: self.kv[dict.key] = list() self.kv[dict.key].append(dict.value) self.kv[dict.key] = self.perform_syntactic_reconcilation( self.kv[dict.key]) dict = Request("ACK-PUT", None, None, dict.request) Messaging.send_message(self, PORT_TO_ID[addr[1]], dict) def perform_get(self, dict): key = dict.key preference_list = self.hash_ring.get_node(key, self.failed_nodes) if (self not in preference_list): coordinator = preference_list[0].id dict = Request("FORWARD-GET", dict.key, dict.value, generate_random_number(), dict.client) Messaging.send_message(self, coordinator, dict) time.sleep(3) if REQUESTS.get(dict.request, False) != True: #print("Timedout GET") dict.action = "GET" dict.request = generate_random_number() self.failed_nodes.append(coordinator) self.perform_get(dict) else: dict.request = generate_random_number() Messaging.broadcast_get(self, preference_list, dict) def retreive_key(self, *data): time.sleep(random.randint(0, 1000) / 1000) dict = data[0] from_node = data[1] val = self.kv.get(dict.key, None) response = Request("ACK-GET", dict.key, val, dict.request) Messaging.send_message(self, from_node, response) #Utility #Print function def string(self, dict): temp = "{" for key in dict: temp = temp + "," + str(key) + ":" temp2 = "[" for val in dict[key]: temp2 += str(val[0]) + "," + str(val[1].clock) + "," temp2 += "]" temp += temp2 temp += "}" return temp # Message gets uncompressed first def run(self): try: while True: data, addr = self.socket.recvfrom(MAX_MESSAGE_SIZE) dict = pickle.loads(data) # #print(self.id + " received data from" + str(addr[1])) if self.state == 1: if dict.action == "PUT": #print(self.id+" received PUT "+str(dict.key)+" "+str(dict.value)+" "+str(dict.client)) thread1 = Thread(target=self.perform_put, args=(dict, )) thread1.start() if dict.action == "STORE": #print(self.id + " received STORE"+str(dict.key)+" "+str(dict.value[0])) thread2 = Thread(target=self.perform_store, args=([dict, addr], )) thread2.start() #print(self.id+" "+self.string(self.kv)) if dict.action == "GET": #print(self.id+" Received Get") thread3 = Thread(target=self.perform_get, args=(dict, )) thread3.start() if dict.action == "FETCH": #print(self.id+" Received Fetch") thread4 = Thread(target=self.retreive_key, args=[dict, PORT_TO_ID[addr[1]]]) thread4.start() if dict.action == "EXIT": #print(self.id+" Will Fail now") self.state = 0 if dict.action == "FORWARD-PUT": #print(self.id+"Received Forward") dict.action = "PUT" response = Request("ACK-FORWARD-PUT", None, None, dict.request) self.socket.sendto(pickle.dumps(response), addr) thread5 = Thread(target=self.perform_put, args=(dict, )) thread5.start() #self.perform_put(dict) if dict.action == "FORWARD-GET": #print(self.id+"Received Forward") dict.action = "GET" response = Request("ACK-FORWARD-GET", None, None, dict.request) self.socket.sendto(pickle.dumps(response), addr) threada = Thread(target=self.perform_get, args=[dict]) threada.start() if dict.action == "PING": #print(self.id+"Received Ping") response = Request("PONG", None) Messaging.send_message(self, PORT_TO_ID[addr[1]], response) if dict.action == "ACK-FORWARD-PUT": #print("Received forward PUT ack") REQUESTS[dict.request] = True if dict.action == "ACK-FORWARD-GET": #print("Received forward Get ack") REQUESTS[dict.request] = True if dict.action == "ACK-PUT": #print("PUT ACK received by "+self.id +" from "+ PORT_TO_ID[addr[1]]) if dict.request not in HISTORY: HISTORY[dict.request] = set() HISTORY[dict.request].add(PORT_TO_ID[addr[1]]) if dict.action == "ACK-GET": #print("GET ACK received by "+self.id +" from "+ PORT_TO_ID[addr[1]]) if dict.request not in HISTORY: HISTORY[dict.request] = list() HISTORY[dict.request].append( (PORT_TO_ID[addr[1]], dict.value)) if dict.action == "SYNC": #print(self.id+"Received handoff Sync") thread10 = Thread(target=self.synchronize, args=[dict]) thread10.start() response = Request("SYNC-ACK", None) Messaging.send_message(self, PORT_TO_ID[addr[1]], response) if (dict.action == "SYNC-ACK"): #print(self.id + " Synced " + PORT_TO_ID[addr[1]]) self.sync_kv.pop(PORT_TO_ID[addr[1]], None) self.check_for_sync.remove(PORT_TO_ID[addr[1]]) if (dict.action == "PONG"): self.failed_nodes.remove(PORT_TO_ID[addr[1]]) #print("node" + str(PORT_TO_ID[addr[1]])+" is alive") #print("Failed nodes=" + str(self.failed_nodes)) if (dict.action == "ENTROPY"): ##print(self.id+" received entropy") threadb = Thread(target=self.antientropy, args=[dict]) threadb.start() else: if dict.action == "REVIVE": #print(self.id+" is reviving") self.state = 1 except Exception: self.run()
def checker(self): # If item is ready to be delivered go ahead. # And then check if the limbo item can be delivered. # Else add it to the limbo queue # And ask for the missing elements # Lets go! while True: try: item = json.loads(self.q_rcv.get()) except ValueError: logging.debug("JSON Couldn't load the message!") continue else: #print(item) msg_type = item['type'] # reassemble the vc if needed if msg_type == 'DATA_2ALL' or msg_type == 'DATA_P2P' or msg_type == 'recovery': aux = VectorClock(static=True, max_entries=item['vc']['max_entries'], key=item['vc']['key']) aux.vc = item['vc']['vc'] item['vc'] = aux self.d_stats['last_vc_seen'] = item['vc'] if msg_type == 'recovery': self.d_stats['count_recovery'] += 1 self.d_stats['count_received'] += 1 logging.debug("[stats] (R, " + str(item['src_id']) + ", " + str(self.src_id) + ", " + str(item['vc'].vc[item['src_id']]) + ")") # Lets check if message is in the rer_worker waiting list # if its there we remove it and continue as normal. if self.inRER(item): del self.rer[(item['vc'].vc[item['src_id']], item['src_id'])] self.d_stats['tvop'] += 1 # if the message is duplicated, means that it should already be delivered or in limbo if not self.duplicate(item['vc']): # Its ugly but we can check it here. for , A1, A2, A3 cases. if msg_type == 'recovery': logging.debug( "I'm in receovery!( I know... its not funny)") if self.vc.vc[item['src_id']] >= item['vc'].vc[ item['src_id']]: self.d_stats['count_fp'] += 1 if self.vc.happened_before(item['vc']): # item can be delivered. self.vc.inc(item['vc'].key) self.q_deliv.put(item['msg']) #logging.debug("[stats] (D, " + str(item['src_id']) + ", " + str(self.src_id) + ", " + str(item['vc'].vc[item['src_id']]) + ")") if msg_type == 'recovery': # the message has been recovered. with the recovery logging.debug( "[stats] (DR, " + str(item['src_id']) + ", " + str(self.src_id) + ", " + str(item['vc'].vc[item['src_id']]) + ")") else: logging.debug( "[stats] (D, " + str(item['src_id']) + ", " + str(self.src_id) + ", " + str(item['vc'].vc[item['src_id']]) + ")") # Now Check if limbo stuff can be delivered. mark_for_deletion = list() len_i = len(self.l_limbo) for i in range(len_i): item = self.l_limbo[i] if self.vc.happened_before(item['vc']): self.vc.inc(item['vc'].key) self.q_deliv.put(item['msg']) logging.debug( "[stats] (D, " + str(item['src_id']) + ", " + str(self.src_id) + ", " + str(item['vc'].vc[item['src_id']]) + ")") # its been delivered, so leave the limbo #del self.l_limbo[i] mark_for_deletion.append(i) else: # ask again self.check_missing(item) for i in reversed(mark_for_deletion): del self.l_limbo[i] else: #it couldnt be delivered, so add to limbo and ask for missing msgs self.l_limbo.append(item) self.d_stats["count_limbo"] += 1 self.check_missing(item) else: if msg_type == 'recovery': self.d_stats['count_fp'] += 1 elif msg_type == 'DATA_2ALL' or msg_type == 'DATA_P2P': self.d_stats['count_A3'] += 1 # It was a duplicate but... you can check if something can be delivered. # Check in the limbo self.check_limbo() elif msg_type == 'ack': # What do we want to do with ack?? self.d_stats['count_ack'] += 1 elif msg_type == 'missing': # We send what they want. missing_msg = self.log[item['num']] self.snd.recovery(dst_id=item['src_id'], str_msg=missing_msg['msg'], vc=missing_msg['vc']) self.d_stats['count_missing'] += 1 elif msg_type == 'nack': # Something went wrong with the AR. self.d_stats['count_nack'] += 1 elif msg_type == 'debug': print(item['msg']) elif msg_type == 'ack_bj': # We update the membership dict with what was sent. self.d_stats['membership'] = item['msg'] # If dst_id exists but the address has changed, then we are screwed. for dst_id, address in self.d_stats['membership'].items(): if not dst_id == self.src_id: if not dst_id in self.d_stats['ms_conns']: # We have to create the connection to this address. sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) sock.connect((address, int(self.d_stats['config'] ['default']['ServerPort']))) self.d_stats['ms_conns'][dst_id] = sock v_fileno = list() v_id = list() for dst_id, conn in self.d_stats['ms_conns'].items(): v_fileno.append(conn.fileno()) v_id.append(dst_id) self.d_stats['ms_fileno'] = v_fileno self.d_stats['ms_fileno_id'] = v_id self.q_rcv.task_done()