class FrontEnd: ''' Class for Front End Server within the distributed system. ''' def __init__(self): self.servers = [] self.rm = None # Initial selection of replica manager to communicate with try: self.rm = self._choose_replica() except ValueError as e: print(e) self.ts = VectorClock(REPLICA_NUM) # Vector timestamp of front end def send_request(self, request): ''' Method invoked by client to send a request. Params: (tuple) request: command to execute and arguments for the command Returns: If the request is a query, return the results of the query, otherwise a confirmation message. ''' r_type = self._request_type(request) # Find a replica manager to send request to if the original is # unavailable if self.rm is not None: try: rm_status = self.rm.get_status() print(rm_status) if rm_status == Status.OFFLINE.value: self.rm = self._choose_replica() except Pyro4.errors.ConnectionClosedError: self.rm = self._choose_replica() else: self.rm = self._choose_replica() if r_type == RType.UPDATE: rm_ts = self.rm.send_update(request, self.ts.value(), str(uuid.uuid4())) print('Update sent: ', request) self.ts.merge(VectorClock.fromiterable(rm_ts)) print('Front end timestamp: ', self.ts.value()) return 'Update submitted!' elif r_type == RType.QUERY: val, rm_ts = self.rm.send_query(request, self.ts.value()) print('Query sent: ', request) self.ts.merge(VectorClock.fromiterable(rm_ts)) print('Front end timestamp: ', self.ts.value()) return val def _choose_replica(self): ''' Select a replica manager to communicate with. Return: Remote object for a replica manager ''' for server in self.servers: server._pyroRelease() self.servers = self._find_replicas() stat = {server: server.get_status() for server in self.servers} available = [] num_offline = list(stat.values()).count(Status.OFFLINE.value) num_active = list(stat.values()).count(Status.ACTIVE.value) if num_active > 0: available = [ k for k in stat.keys() if stat[k] == Status.ACTIVE.value ] elif num_offline == len(self.servers): raise Exception('All servers offline') else: available = [ k for k in stat.keys() if stat[k] != Status.OFFLINE.value ] if not available: return None return random.choice(available) @staticmethod def _request_type(request): ''' Determine whether a request is an update or query. Params: (tuple) request: request to check Returns: Enum representing the type of request ''' op = request[0] op_type = op.split('.')[0] if op_type == 'u': return RType.UPDATE if op_type == 'q': return RType.QUERY raise ValueError('command not recognised') @staticmethod def _find_replicas(): ''' Find all online replica managers. Returns: servers: list of remote server objects for replica managers ''' servers = [] with Pyro4.locateNS() as ns: for server, uri in ns.list(prefix="network.replica.").items(): print("found replica", server) servers.append(Pyro4.Proxy(uri)) if not servers: raise ValueError( "No servers found! (are the movie servers running?)") return servers[:REPLICA_NUM]
class ReplicaManager(threading.Thread): ''' Class for a Replica Server within the distributed system, implementing the gossip architecture. ''' def __init__(self, replica_id, stopper, status=None): super().__init__() self._id = replica_id # Replica status properties self.failure_prob = 0.1 self.overload_prob = 0.2 self.auto_status = True if status not in [n.value for n in list(Status)]: print('Invalid status provided, defaulting to active.') self.status = Status.ACTIVE else: self.status = Status(status) self.auto_status = False print(f'Status set to {status}.', 'Automatic status updating disabled.') # Gossip Architecture State self.value_ts = VectorClock(REPLICA_NUM) # aka data timestamp self.replica_ts = VectorClock(REPLICA_NUM) # aka log timestamp self.update_log = [] self.ts_table = [ VectorClock(REPLICA_NUM) if i != self._id else None for i in range(REPLICA_NUM) ] self.executed = [] self.pending_queries = queue.Queue() self.query_results = {} self.interval = 8.0 # interval between gossip exchanges self.other_replicas = self._find_replicas() self.stopper = stopper # Used to indicate to server to stop # Locks for objects shared between threads self.vts_lock = threading.Lock() # for value_ts self.rts_lock = threading.Lock() # for replica_ts self.log_lock = threading.Lock() # for update_log def run(self): ''' Override of threading.Thread run() method. Sends gossip to other replica managers periodically. ''' while not self.stopper.is_set(): if self.status != Status.OFFLINE: for r_id, rm in self.other_replicas: rm._pyroRelease() self.other_replicas = self._find_replicas() with self.rts_lock: print('\n--- SENDING GOSSIP ---') for r_id, rm in self.other_replicas: r_ts = self.ts_table[r_id] m_log = self._get_recent_updates(r_ts) print(f'Updates to send to RM {r_id}: ', m_log) try: rm.send_gossip(m_log, self.replica_ts.value(), self._id) print(f'Gossip sent to RM {r_id}') except Pyro4.errors.CommunicationError as e: print(f'Failed to send gossip to RM {r_id}') print('----------------------') if self.auto_status: self._update_status() print('Status: ', self.status.value, '\n') self.stopper.wait(self.interval) print('Stopper set, gossip thread stopping.') def send_query(self, q_op, q_prev): ''' Method invoked by the front end to send a query. Params: (string) q_op: query command (tuple) q_prev: vector timestamp of front end Returns: response: results of query ''' print('Query received: ', q_op, q_prev) response = None q_prev = VectorClock.fromiterable(q_prev) # stable = are we up to date enough to handle the query correctly? stable = False with self.vts_lock: if q_prev <= self.value_ts: # stability criteria for query val = self._apply_query(q_op) new = self.value_ts.value() response = (val, new) stable = True print('Value timestamp: ', self.value_ts.value(), '\n') if not stable: # if not stable, add to a dictionary of pending queries and wait self.query_results[(q_op, q_prev.value())] = queue.Queue(maxsize=1) self.pending_queries.put((q_op, q_prev)) # Wait for query to be executed after some gossip exchange response = self.query_results[(q_op, q_prev.value())].get() # Remove entry from pending query dictionary del self.query_results[(q_op, q_prev.value())] return response def send_update(self, u_op, u_prev, u_id): ''' Method invoked by the front end to send an update. Params: (string) u_op: update command (tuple) u_prev: vector timestamp of front end (string) u_id: unique ID for update Returns: ts: timestamp representing having executed the update or None if the update has already been executed ''' print('Update received: ', u_op, u_prev, u_id) ts = None # Add update to log if it hasn't already been executed if u_id not in self.executed: with self.rts_lock: self.replica_ts.increment(self._id) ts = list(u_prev[:]) ts[self._id] = self.replica_ts.value()[self._id] print('Replica timestamp: ', self.replica_ts, '\n') ts = VectorClock.fromiterable(ts) u_prev = VectorClock.fromiterable(u_prev) log_record = (self._id, ts, u_op, u_prev, u_id) with self.log_lock: self.update_log.append(log_record) print('Update record: ', log_record) # Execute update if it is stable with self.vts_lock: if u_prev <= self.value_ts: # stability criteria for query self._execute_update(u_op, u_id, ts) return ts.value() return ts @Pyro4.oneway def send_gossip(self, m_log, m_ts, r_id): ''' Method invoked by other replica managers to send gossip. Params: (string) m_log: recent updates from replica manager (tuple) m_ts: log timestamp of sending replica manager (string) r_id: ID of sending replica manager Returns: ts: timestamp representing having executed the update or None if the update has already been executed ''' if self.status != Status.OFFLINE: print('\n--- RECEIVING GOSSIP ---') print(f'Gossip received from RM {r_id}') print(m_ts) print(m_log) print() # Merge m_log into update log self._merge_update_log(m_log) # Merge our replica timestamp with m_ts m_ts = VectorClock.fromiterable(m_ts) with self.rts_lock: self.replica_ts.merge(m_ts) print('Replica timestamp: ', self.replica_ts) # Execute all updates that have now become stable stable = self._get_stable_updates() for update in stable: _id, ts, u_op, u_prev, u_id = update with self.vts_lock: self._execute_update(u_op, u_id, ts) # Set the timestamp of the sending replica manager in our timestamp # table self.ts_table[r_id] = m_ts # Execute all stable pending queries while True: try: q_op, q_prev = self.pending_queries.get(block=False) with self.vts_lock: if q_prev <= self.value_ts: val = self._apply_query(q_op) new = self.value_ts.value() self.query_results[(q_op, q_prev.value())].put( (val, new)) except queue.Empty: break print('------------------------') def get_status(self): ''' Method invoked by front end to query the server status. Returns: status of the server ''' return self.status.value def set_status(self, status): ''' Method invoked by status_control.py to set the server status. ''' self.status = Status(status) def toggle_auto_status(self, auto): ''' Method invoked by status_control.py to set the server status to update automatically or not. ''' if auto: self.auto_status = True else: self.auto_status = False def _update_status(self): ''' Set the server status probabilistically. ''' overloaded = random.random() failed = random.random() if failed < self.failure_prob: self.status = Status.OFFLINE elif overloaded < self.overload_prob: self.status = Status.OVERLOADED else: self.status = Status.ACTIVE def _apply_query(self, q_op): ''' Execute a query command. Params: (string) q_op: query command to execute Returns: val: result of query ''' print('Query applied. ', q_op, '\n') val = None op, *params = q_op query = self._parse_q_op(op) val = query(*params) return val def _apply_update(self, u_op): ''' Execute an update command. Params: (string) u_op: update command to execute ''' print('Update applied.', u_op, '\n') op, *params = u_op update = self._parse_u_op(op) update(*params) def _execute_update(self, u_op, u_id, ts): ''' Execute an update. Params: (string) u_op: update command to execute (string) u_id: ID of update to execute (VectorClock) ts: timestamp of update to execute ''' # Return immediately if update has already been executed if u_id in self.executed: return self._apply_update(u_op) # Execute the update self.value_ts.merge(ts) # Update the value timestamp self.executed.append(u_id) # Add update to executed updates print('Value timestamp: ', self.value_ts) def _merge_update_log(self, m_log): ''' Merge the update log with updates from a gossip message. Params: m_log: list of updates from a gossip message ''' for record in m_log: _id, ts, u_op, u_prev, u_id = record ts = VectorClock.fromiterable(ts) u_prev = VectorClock.fromiterable(u_prev) with self.rts_lock, self.log_lock: new_record = (_id, ts, u_op, u_prev, u_id) if new_record not in self.update_log: if not ts <= self.replica_ts: self.update_log.append(new_record) def _get_stable_updates(self): ''' Retrieve all stable updates from the update log. Returns: stable: list of updates that can be executed. ''' stable = [] with self.vts_lock, self.log_lock: stable = [ record for record in self.update_log if record[3] <= self.value_ts ] stable.sort(key=lambda r: r[3]) return stable def _get_recent_updates(self, r_ts): ''' Retrieve updates from update log that are more recent than our recorded value of the timestamp of another replica manager. Params: (VectorClock) r_ts: Timestamp of another replica manager, sent in gossip Returns: recent: all updates from update log that are more recent than the given timestamp ''' recent = [] with self.log_lock: for record in self.update_log: _id, ts, u_op, u_prev, u_id = record if ts > r_ts: new_record = (_id, ts.value(), u_op, u_prev.value(), u_id) recent.append(new_record) return recent def _find_replicas(self): ''' Find all online replica managers. Returns: servers: list of remote server objects for replica managers ''' servers = [] try: with Pyro4.locateNS() as ns: for server, uri in ns.list(prefix="network.replica.").items(): server_id = int(server.split('.')[-1]) if server_id != self._id: servers.append((server_id, Pyro4.Proxy(uri))) except Pyro4.errors.NamingError: print('Could not find Pyro nameserver.') servers.sort() return servers[:REPLICA_NUM] @staticmethod def _parse_q_op(op): ''' Match query command strings with query functions. Params: (string) op: query command Returns: function corresponding to the query command ''' return { ROp.GET_AVG_RATING.value: get_avg_movie_rating, ROp.GET_RATINGS.value: get_movie_ratings, ROp.GET_GENRES.value: get_movie_genres, ROp.GET_MOVIE.value: get_movie_by_title, ROp.GET_TAGS.value: get_movie_tags, ROp.SEARCH_TITLE.value: search_by_title, ROp.SEARCH_GENRE.value: search_by_genre, ROp.SEARCH_TAG.value: search_by_tag }[op] @staticmethod def _parse_u_op(op): ''' Match update command strings with update functions. Params: (string) op: update command Returns: function corresponding to the update command ''' return { ROp.ADD_RATING.value: submit_rating, ROp.ADD_TAG.value: submit_tag }[op]
class Node(KV_store): '''docstring for node class''' def __init__(self, router, address, view, replication_factor): self.gossiping = False self.sched = Scheduler() self.sched.start() KV_store.__init__(self, address) self.history = [('Initialized', datetime.now())] self.ADDRESS = address self.VC = VectorClock(view=view, clock=None) self.ring_edge = 691 if len( view) < 100 else 4127 # parameter for hash mod value self.repl_factor = replication_factor self.num_shards = 0 self.virtual_range = 10 self.shard_interval = self.ring_edge // self.virtual_range self.nodes = [] self.shard_ID = -1 self.V_SHARDS = [] # store all virtual shards self.P_SHARDS = [[] for i in range(0, self.num_shards) ] # map physical shards to nodes self.virtual_translation = {} # map virtual shards to physical shards self.backoff_mod = 113 self.router = router self.view_change(view, replication_factor) def __repr__(self): return { 'ADDRESS': self.ADDRESS, 'V_SHARDS': self.V_SHARDS, 'P_SHARDS': self.P_SHARDS, 'KEYS': len(self.keystore) } def __str__(self): return 'ADDRESS: ' + self.ADDRESS + '\nREPL_F: ' + str( self.repl_factor) + '\nNODES: ' + (', '.join(map( str, self.nodes))) + '\nP_SHARDS: ' + (', '.join( map(str, self.P_SHARDS))) ''' give a state report this includes node data and distribution of keys to nodes ''' def state_report(self): state = self.__repr__() state['HISTORY'] = {} string = 'node' itr = 1 for event in self.history: key = string + str(itr) itr += 1 state['HISTORY'][key] = event return state ''' return all physical shards ''' def all_shards(self): return self.P_SHARDS def all_nodes(self): return self.nodes ''' get all nodes in this shard ''' def shard_replicas(self, shard_ID): return self.P_SHARDS[shard_ID] ''' hash frunction is a composit of xxhash modded by prime ''' def hash(self, key, Type): hash_val = hasher.xxh32(key).intdigest() # may be expensive but will produce better distribution return (hash_val % self.ring_edge) ''' evenly distribute nodes into num_shard buckets ''' def even_distribution(self, repl_factor, nodes): nodes.sort() num_shards = (len(nodes) // repl_factor) replicas = (len(nodes) // num_shards) overflow = (len(nodes) % num_shards) shards = [[] for i in range(0, num_shards)] shard_dict = {} node_iter = 0 for shard in range(num_shards): extra = (1 if shard < overflow else 0) interval = replicas + extra shards[shard] = nodes[node_iter:(node_iter + interval)] node_iter += interval for node in shards[shard]: shard_dict[node] = shard return shard_dict ''' Perform a key operation, ie. find the correct shard given key. First hash the key then perform binary search to find the correct shard to store the key. ''' def find_match(self, key): ring_val = self.hash(key, 'consistent') # get the virtual shard number v_shard = self.find_shard('predecessor', ring_val) # convert to physical shard shard_ID = self.virtual_translation[v_shard] return shard_ID ''' perform binary search on list of virtual shards given ring value we need to be careful about wrap around case. If ring_val >= max_ring_val, return 0 ''' def find_shard(self, direction, ring_val): if direction == 'predecessor': v_shard = bisect_left(self.V_SHARDS, ring_val) if v_shard: return self.V_SHARDS[v_shard - 1] return self.V_SHARDS[-1] elif direction == 'successor': v_shard = bisect_right(self.V_SHARDS, ring_val) if v_shard != len(self.V_SHARDS): return self.V_SHARDS[v_shard] return self.V_SHARDS[0] ''' respond to view change request, perform a reshard this can only be done if all nodes have been given new view 2 cases: 1. len(nodes) + 1 // r > or < shard_num: we need to add or remove a shard to maintain repl_factor 2. add and/or remove nodes ''' def view_change(self, view, repl_factor): new_num_shards = len(view) // repl_factor if new_num_shards == 1: new_num_shards = 2 view.sort() buckets = self.even_distribution(repl_factor, view) #print('buckets', buckets) # add nodes and shards for node in view: my_shard = buckets[node] if node == self.ADDRESS: self.shard_ID = buckets[node] self.sched.add_interval_job(self.gossip, seconds=self.gossip_backoff()) # add a new node if node not in self.nodes: self.add_node(node, my_shard, new_num_shards) # move node to new shard else: if my_shard >= len(self.P_SHARDS): self.add_shard() if node not in self.P_SHARDS[my_shard]: self.move_node(node, my_shard) old_nodes = list(set(self.nodes) - set(view)) # remove nodes from view for node in old_nodes: self.remove_node(node) # remove empty shards for shard_ID in range(0, len(self.P_SHARDS)): if len(self.P_SHARDS[shard_ID]) == 0: self.remove_shard(shard_ID) ''' Add a single node to shards and get keys from shard replicas ''' def add_node(self, node, shard_ID, num_shards): # do we need to add another shard before adding nodes while num_shards > self.num_shards: self.add_shard() # update internal data structures self.nodes.append(node) self.nodes.sort() self.P_SHARDS[shard_ID].append(node) # determine if the node's shard is this shard if self.shard_ID == shard_ID: #print('<adding node to:', shard_ID) self.shard_keys() ''' move node from old shard to new shard and perform atomic key transfer ''' def move_node(self, node, shard_ID): old_shard_ID = self.nodes.index(node) // self.num_shards if node not in self.P_SHARDS[old_shard_ID]: if old_shard_ID > 0 and node in self.P_SHARDS[old_shard_ID - 1]: old_shard_ID += -1 else: old_shard_ID += 1 # do we need to add another shard before adding nodes while shard_ID > len(self.P_SHARDS): self.add_shard() self.atomic_key_transfer(old_shard_ID, shard_ID, node) self.P_SHARDS[shard_ID].append(node) self.P_SHARDS[old_shard_ID].pop( self.P_SHARDS[old_shard_ID].index(node)) ''' remove single node from a shard and send final state to shard replicas ''' def remove_node(self, node): shard_ID = (self.nodes.index(node) - 1) // self.num_shards if shard_ID > 0 and shard_ID < len( self.P_SHARDS) and node not in self.P_SHARDS[shard_ID]: if shard_ID > 0 and node in self.P_SHARDS[shard_ID - 1]: shard_ID += -1 else: shard_ID += 1 #print('error finding node') if node == self.ADDRESS: print('<send my final state to my replicas before removing') success = self.final_state_transfer(node) if success: self.nodes.pop(self.nodes.index(node)) else: raise Exception('<final_state_transfer failed>') else: self.nodes.pop(self.nodes.index(node)) self.P_SHARDS[shard_ID].pop(self.P_SHARDS[shard_ID].index(node)) ''' add shard to view ''' def add_shard(self): new_shards = [] p_shard = self.num_shards if p_shard >= len(self.P_SHARDS): self.P_SHARDS.append([]) for v_shard in range(self.virtual_range): virtural_shard = str(p_shard) + str(v_shard) ring_num = self.hash(virtural_shard, 'consistent') # unique value on 'ring' # if ring_num is already in unsorted list, skip this iteration if ring_num in self.V_SHARDS: #print('<System: Hash collision detected>') continue self.V_SHARDS.append(ring_num) self.virtual_translation[ring_num] = p_shard self.num_shards += 1 self.V_SHARDS.sort() return new_shards ''' remove from all internal data structures if there are no nodes in shard ''' def remove_shard(self, shard_ID): self.P_SHARDS.pop(shard_ID) ''' get all keys for a given shard ''' def shard_keys(self): pass ''' perform an atomic key transfer concurrent operation: get new keys, send old keys, delete old keys ''' def atomic_key_transfer(self, old_shard_ID, new_shard_ID, node): # message all nodes and tell them your state # get new keys from new replica self.final_state_transfer() old_kv = self.KV_store for replica in self.P_SHARDS[old_shard_ID]: data = None try: res, status_code = self.router.GET(replica, '/kv-store/internal/KV', data, False) except: continue if status_code == 201: new_kv = res.get('KV_store') update = False for key in new_kv: self.KV_store.keystore[key] = new_kv[key] for key in old_kv: del self.KV_store.keystore[key] return True return False ''' send final state of node before removing a node ''' def final_state_transfer(self, node): data = {"kv-store": self.keystore, "context": self.VC.__repr__()} replica_ip_addresses = self.shard_replicas(self.shard_ID) for replica in replica_ip_addresses: if (replica != self.ADDRESS): try: res, status_code = self.router.PUT( replica, '/kv-store/internal/state-transfer', data, False) except: continue if status_code == 201: return True return False ''' handle node failures, check if node should be removed or not ''' def handle_unresponsive_node(self, node): pass def gossip_backoff(self): return hash(self.ADDRESS) % random.randint(20, 40) def gossip(self): if (self.gossiping == False): current_key_store = self.keystore self.gossiping = True replica_ip_addresses = self.shard_replicas(self.shard_ID) replica = replica_ip_addresses[(random.randint( 0, len(replica_ip_addresses) - 1))] while (self.ADDRESS == replica): replica = replica_ip_addresses[(random.randint( 0, len(replica_ip_addresses) - 1))] myNumber = int((self.ADDRESS.split(".")[3]).split(":")[0]) otherNumber = int((replica.split(".")[3]).split(":")[0]) tiebreaker = replica if (otherNumber > myNumber) else self.ADDRESS data = { "context": self.VC.__repr__(), "kv-store": current_key_store, "tiebreaker": tiebreaker } print("sending to node: " + replica + " " + str(data), file=sys.stderr) try: response = self.router.PUT(replica, '/kv-store/internal/gossip/', json.dumps(data)) except: code = -1 code = response.status_code if (code == 200): # 200: They took my data self.gossiping = False elif (code == 501): content = response.json() # 501: # the other node was either the tiebreaker or happened after self # so this node takes its data # context of node other_context = content["context"] # key store of incoming node trying to gossip other_kvstore = content["kv-store"] incoming_Vc = VectorClock(view=None, clock=other_context) if bool(other_kvstore) and not incoming_Vc.allFieldsZero(): if current_key_store == self.keystore: print("I TOOK DATA: " + str(self.keystore), file=sys.stderr) self.VC.merge(other_context, self.ADDRESS) self.keystore = other_kvstore else: print("I RECIEVED AN UPDATE WHILE GOSSIPING, ABORT", file=sys.stderr) self.gossip = False #self happened before other, take its kvstore and merge with my clock # concurrent but other is tiebreaker else: # 400: Other is already gossiping with someone else # ELSE: unresponsive node (maybe itll be code 404?) self.gossiping = False else: # Curretly gossiping, # Will call after gossip backoff again self.gossiping = False return 200