def __init__(self, id): self.vector_clock = VectorClock() #state of the process, 1 means alive, 0 means dead self.state = 1 self.message_queue = Queue() self.kv = {} self.count = 0 self.vector_clock.update(id, self.count) self.id = id self.hash_ring = ConsistentHashRing.getInstance() self.hash_ring.add_node(self) self.read_port = CONFIG[self.id][1] self.write_port = CONFIG[self.id][2] self.ip = CONFIG[self.id][0] self.socket = socket(AF_INET, SOCK_DGRAM) self.socket.bind((self.ip, self.read_port)) self.pending_reads = {} self.pending_writes = {} self.N = 3 self.R = 1 self.W = 2 self.failed_nodes = [] self.sync_kv = {} self.check_for_sync = [] self.back = Thread(target=self.background_process) self.history = {} Thread.__init__(self) self.back.start()
def perform_semantic_reconcilation(self, data): result = -INF vc = VectorClock() if len(data) > 1: clocks = [] for value, clock in data: result = max(result, value) clocks.append(clock) return (result, vc.converge(clocks)) else: return (data[0][0], data[0][1])
def __init__(self, server=None, user_id=0, initial_version=None, initial={}): self.state = initial self.initial_state = initial self.history = History() self.user_id = user_id self.version = VectorClock(user_id, initial_version) self.server = server if server is not None: self.server.on_recieve += self.on_recieve self.on_update = Callback()
def put(self, key, value, retries=0): while retries < MAX_RETRIES: dynamo_node = self.nodes[random.randint(0, len(self.nodes) - 1)] vc = VectorClock() vc.update(dynamo_node, 0) value1 = (value, self.datastore.get(key, vc)) #print("Client value is "+str(value1)) put_request = Request("PUT", key, value1, None, self.port) self.socket.settimeout(5) Messaging.send_message(self, dynamo_node, put_request) try: while True: data, addr = self.socket.recvfrom(40960) if data: self.socket.settimeout(None) return pickle.loads(data) except Exception: self.socket.settimeout(None) retries += 1
if config['default']['SocketType'] == 'udp': sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) elif config['default']['SocketType'] == 'tcp': sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((config['default']['ServerAddress'], int(config['default']['ServerPort']))) else: print("Error: SocketType is neither udp or tcp.") exit(1) d_stats = dict() d_stats['t_init'] = int(time.strftime("%s")) vc = VectorClock(static=bool(config['VectorClock']['Static']), max_entries=int(config['VectorClock']['MaxEntries']), key=clientId) log = list() # To append and recorder all sent operations. snd = Snd(clientId, host=config['default']['ServerAddress'], port=int(config['default']['ServerPort']), sock=sock, vc=vc, log=log, stats=d_stats, config=config) rcv = Rcv(clientId, host=config['default']['ServerAddress'], port=int(config['default']['ServerPort']), sock=sock,
def checker(self): # If item is ready to be delivered go ahead. # And then check if the limbo item can be delivered. # Else add it to the limbo queue # And ask for the missing elements # Lets go! while True: try: item = json.loads(self.q_rcv.get()) except ValueError: logging.debug("JSON Couldn't load the message!") continue else: #print(item) msg_type = item['type'] # reassemble the vc if needed if msg_type == 'DATA_2ALL' or msg_type == 'DATA_P2P' or msg_type == 'recovery': aux = VectorClock(static=True, max_entries=item['vc']['max_entries'], key=item['vc']['key']) aux.vc = item['vc']['vc'] item['vc'] = aux self.d_stats['last_vc_seen'] = item['vc'] if msg_type == 'recovery': self.d_stats['count_recovery'] += 1 self.d_stats['count_received'] += 1 logging.debug("[stats] (R, " + str(item['src_id']) + ", " + str(self.src_id) + ", " + str(item['vc'].vc[item['src_id']]) + ")") # Lets check if message is in the rer_worker waiting list # if its there we remove it and continue as normal. if self.inRER(item): del self.rer[(item['vc'].vc[item['src_id']], item['src_id'])] self.d_stats['tvop'] += 1 # if the message is duplicated, means that it should already be delivered or in limbo if not self.duplicate(item['vc']): # Its ugly but we can check it here. for , A1, A2, A3 cases. if msg_type == 'recovery': logging.debug( "I'm in receovery!( I know... its not funny)") if self.vc.vc[item['src_id']] >= item['vc'].vc[ item['src_id']]: self.d_stats['count_fp'] += 1 if self.vc.happened_before(item['vc']): # item can be delivered. self.vc.inc(item['vc'].key) self.q_deliv.put(item['msg']) #logging.debug("[stats] (D, " + str(item['src_id']) + ", " + str(self.src_id) + ", " + str(item['vc'].vc[item['src_id']]) + ")") if msg_type == 'recovery': # the message has been recovered. with the recovery logging.debug( "[stats] (DR, " + str(item['src_id']) + ", " + str(self.src_id) + ", " + str(item['vc'].vc[item['src_id']]) + ")") else: logging.debug( "[stats] (D, " + str(item['src_id']) + ", " + str(self.src_id) + ", " + str(item['vc'].vc[item['src_id']]) + ")") # Now Check if limbo stuff can be delivered. mark_for_deletion = list() len_i = len(self.l_limbo) for i in range(len_i): item = self.l_limbo[i] if self.vc.happened_before(item['vc']): self.vc.inc(item['vc'].key) self.q_deliv.put(item['msg']) logging.debug( "[stats] (D, " + str(item['src_id']) + ", " + str(self.src_id) + ", " + str(item['vc'].vc[item['src_id']]) + ")") # its been delivered, so leave the limbo #del self.l_limbo[i] mark_for_deletion.append(i) else: # ask again self.check_missing(item) for i in reversed(mark_for_deletion): del self.l_limbo[i] else: #it couldnt be delivered, so add to limbo and ask for missing msgs self.l_limbo.append(item) self.d_stats["count_limbo"] += 1 self.check_missing(item) else: if msg_type == 'recovery': self.d_stats['count_fp'] += 1 elif msg_type == 'DATA_2ALL' or msg_type == 'DATA_P2P': self.d_stats['count_A3'] += 1 # It was a duplicate but... you can check if something can be delivered. # Check in the limbo self.check_limbo() elif msg_type == 'ack': # What do we want to do with ack?? self.d_stats['count_ack'] += 1 elif msg_type == 'missing': # We send what they want. missing_msg = self.log[item['num']] self.snd.recovery(dst_id=item['src_id'], str_msg=missing_msg['msg'], vc=missing_msg['vc']) self.d_stats['count_missing'] += 1 elif msg_type == 'nack': # Something went wrong with the AR. self.d_stats['count_nack'] += 1 elif msg_type == 'debug': print(item['msg']) elif msg_type == 'ack_bj': # We update the membership dict with what was sent. self.d_stats['membership'] = item['msg'] # If dst_id exists but the address has changed, then we are screwed. for dst_id, address in self.d_stats['membership'].items(): if not dst_id == self.src_id: if not dst_id in self.d_stats['ms_conns']: # We have to create the connection to this address. sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) sock.connect((address, int(self.d_stats['config'] ['default']['ServerPort']))) self.d_stats['ms_conns'][dst_id] = sock v_fileno = list() v_id = list() for dst_id, conn in self.d_stats['ms_conns'].items(): v_fileno.append(conn.fileno()) v_id.append(dst_id) self.d_stats['ms_fileno'] = v_fileno self.d_stats['ms_fileno_id'] = v_id self.q_rcv.task_done()