class ReplicaManager(threading.Thread): ''' Class for a Replica Server within the distributed system, implementing the gossip architecture. ''' def __init__(self, replica_id, stopper, status=None): super().__init__() self._id = replica_id # Replica status properties self.failure_prob = 0.1 self.overload_prob = 0.2 self.auto_status = True if status not in [n.value for n in list(Status)]: print('Invalid status provided, defaulting to active.') self.status = Status.ACTIVE else: self.status = Status(status) self.auto_status = False print(f'Status set to {status}.', 'Automatic status updating disabled.') # Gossip Architecture State self.value_ts = VectorClock(REPLICA_NUM) # aka data timestamp self.replica_ts = VectorClock(REPLICA_NUM) # aka log timestamp self.update_log = [] self.ts_table = [ VectorClock(REPLICA_NUM) if i != self._id else None for i in range(REPLICA_NUM) ] self.executed = [] self.pending_queries = queue.Queue() self.query_results = {} self.interval = 8.0 # interval between gossip exchanges self.other_replicas = self._find_replicas() self.stopper = stopper # Used to indicate to server to stop # Locks for objects shared between threads self.vts_lock = threading.Lock() # for value_ts self.rts_lock = threading.Lock() # for replica_ts self.log_lock = threading.Lock() # for update_log def run(self): ''' Override of threading.Thread run() method. Sends gossip to other replica managers periodically. ''' while not self.stopper.is_set(): if self.status != Status.OFFLINE: for r_id, rm in self.other_replicas: rm._pyroRelease() self.other_replicas = self._find_replicas() with self.rts_lock: print('\n--- SENDING GOSSIP ---') for r_id, rm in self.other_replicas: r_ts = self.ts_table[r_id] m_log = self._get_recent_updates(r_ts) print(f'Updates to send to RM {r_id}: ', m_log) try: rm.send_gossip(m_log, self.replica_ts.value(), self._id) print(f'Gossip sent to RM {r_id}') except Pyro4.errors.CommunicationError as e: print(f'Failed to send gossip to RM {r_id}') print('----------------------') if self.auto_status: self._update_status() print('Status: ', self.status.value, '\n') self.stopper.wait(self.interval) print('Stopper set, gossip thread stopping.') def send_query(self, q_op, q_prev): ''' Method invoked by the front end to send a query. Params: (string) q_op: query command (tuple) q_prev: vector timestamp of front end Returns: response: results of query ''' print('Query received: ', q_op, q_prev) response = None q_prev = VectorClock.fromiterable(q_prev) # stable = are we up to date enough to handle the query correctly? stable = False with self.vts_lock: if q_prev <= self.value_ts: # stability criteria for query val = self._apply_query(q_op) new = self.value_ts.value() response = (val, new) stable = True print('Value timestamp: ', self.value_ts.value(), '\n') if not stable: # if not stable, add to a dictionary of pending queries and wait self.query_results[(q_op, q_prev.value())] = queue.Queue(maxsize=1) self.pending_queries.put((q_op, q_prev)) # Wait for query to be executed after some gossip exchange response = self.query_results[(q_op, q_prev.value())].get() # Remove entry from pending query dictionary del self.query_results[(q_op, q_prev.value())] return response def send_update(self, u_op, u_prev, u_id): ''' Method invoked by the front end to send an update. Params: (string) u_op: update command (tuple) u_prev: vector timestamp of front end (string) u_id: unique ID for update Returns: ts: timestamp representing having executed the update or None if the update has already been executed ''' print('Update received: ', u_op, u_prev, u_id) ts = None # Add update to log if it hasn't already been executed if u_id not in self.executed: with self.rts_lock: self.replica_ts.increment(self._id) ts = list(u_prev[:]) ts[self._id] = self.replica_ts.value()[self._id] print('Replica timestamp: ', self.replica_ts, '\n') ts = VectorClock.fromiterable(ts) u_prev = VectorClock.fromiterable(u_prev) log_record = (self._id, ts, u_op, u_prev, u_id) with self.log_lock: self.update_log.append(log_record) print('Update record: ', log_record) # Execute update if it is stable with self.vts_lock: if u_prev <= self.value_ts: # stability criteria for query self._execute_update(u_op, u_id, ts) return ts.value() return ts @Pyro4.oneway def send_gossip(self, m_log, m_ts, r_id): ''' Method invoked by other replica managers to send gossip. Params: (string) m_log: recent updates from replica manager (tuple) m_ts: log timestamp of sending replica manager (string) r_id: ID of sending replica manager Returns: ts: timestamp representing having executed the update or None if the update has already been executed ''' if self.status != Status.OFFLINE: print('\n--- RECEIVING GOSSIP ---') print(f'Gossip received from RM {r_id}') print(m_ts) print(m_log) print() # Merge m_log into update log self._merge_update_log(m_log) # Merge our replica timestamp with m_ts m_ts = VectorClock.fromiterable(m_ts) with self.rts_lock: self.replica_ts.merge(m_ts) print('Replica timestamp: ', self.replica_ts) # Execute all updates that have now become stable stable = self._get_stable_updates() for update in stable: _id, ts, u_op, u_prev, u_id = update with self.vts_lock: self._execute_update(u_op, u_id, ts) # Set the timestamp of the sending replica manager in our timestamp # table self.ts_table[r_id] = m_ts # Execute all stable pending queries while True: try: q_op, q_prev = self.pending_queries.get(block=False) with self.vts_lock: if q_prev <= self.value_ts: val = self._apply_query(q_op) new = self.value_ts.value() self.query_results[(q_op, q_prev.value())].put( (val, new)) except queue.Empty: break print('------------------------') def get_status(self): ''' Method invoked by front end to query the server status. Returns: status of the server ''' return self.status.value def set_status(self, status): ''' Method invoked by status_control.py to set the server status. ''' self.status = Status(status) def toggle_auto_status(self, auto): ''' Method invoked by status_control.py to set the server status to update automatically or not. ''' if auto: self.auto_status = True else: self.auto_status = False def _update_status(self): ''' Set the server status probabilistically. ''' overloaded = random.random() failed = random.random() if failed < self.failure_prob: self.status = Status.OFFLINE elif overloaded < self.overload_prob: self.status = Status.OVERLOADED else: self.status = Status.ACTIVE def _apply_query(self, q_op): ''' Execute a query command. Params: (string) q_op: query command to execute Returns: val: result of query ''' print('Query applied. ', q_op, '\n') val = None op, *params = q_op query = self._parse_q_op(op) val = query(*params) return val def _apply_update(self, u_op): ''' Execute an update command. Params: (string) u_op: update command to execute ''' print('Update applied.', u_op, '\n') op, *params = u_op update = self._parse_u_op(op) update(*params) def _execute_update(self, u_op, u_id, ts): ''' Execute an update. Params: (string) u_op: update command to execute (string) u_id: ID of update to execute (VectorClock) ts: timestamp of update to execute ''' # Return immediately if update has already been executed if u_id in self.executed: return self._apply_update(u_op) # Execute the update self.value_ts.merge(ts) # Update the value timestamp self.executed.append(u_id) # Add update to executed updates print('Value timestamp: ', self.value_ts) def _merge_update_log(self, m_log): ''' Merge the update log with updates from a gossip message. Params: m_log: list of updates from a gossip message ''' for record in m_log: _id, ts, u_op, u_prev, u_id = record ts = VectorClock.fromiterable(ts) u_prev = VectorClock.fromiterable(u_prev) with self.rts_lock, self.log_lock: new_record = (_id, ts, u_op, u_prev, u_id) if new_record not in self.update_log: if not ts <= self.replica_ts: self.update_log.append(new_record) def _get_stable_updates(self): ''' Retrieve all stable updates from the update log. Returns: stable: list of updates that can be executed. ''' stable = [] with self.vts_lock, self.log_lock: stable = [ record for record in self.update_log if record[3] <= self.value_ts ] stable.sort(key=lambda r: r[3]) return stable def _get_recent_updates(self, r_ts): ''' Retrieve updates from update log that are more recent than our recorded value of the timestamp of another replica manager. Params: (VectorClock) r_ts: Timestamp of another replica manager, sent in gossip Returns: recent: all updates from update log that are more recent than the given timestamp ''' recent = [] with self.log_lock: for record in self.update_log: _id, ts, u_op, u_prev, u_id = record if ts > r_ts: new_record = (_id, ts.value(), u_op, u_prev.value(), u_id) recent.append(new_record) return recent def _find_replicas(self): ''' Find all online replica managers. Returns: servers: list of remote server objects for replica managers ''' servers = [] try: with Pyro4.locateNS() as ns: for server, uri in ns.list(prefix="network.replica.").items(): server_id = int(server.split('.')[-1]) if server_id != self._id: servers.append((server_id, Pyro4.Proxy(uri))) except Pyro4.errors.NamingError: print('Could not find Pyro nameserver.') servers.sort() return servers[:REPLICA_NUM] @staticmethod def _parse_q_op(op): ''' Match query command strings with query functions. Params: (string) op: query command Returns: function corresponding to the query command ''' return { ROp.GET_AVG_RATING.value: get_avg_movie_rating, ROp.GET_RATINGS.value: get_movie_ratings, ROp.GET_GENRES.value: get_movie_genres, ROp.GET_MOVIE.value: get_movie_by_title, ROp.GET_TAGS.value: get_movie_tags, ROp.SEARCH_TITLE.value: search_by_title, ROp.SEARCH_GENRE.value: search_by_genre, ROp.SEARCH_TAG.value: search_by_tag }[op] @staticmethod def _parse_u_op(op): ''' Match update command strings with update functions. Params: (string) op: update command Returns: function corresponding to the update command ''' return { ROp.ADD_RATING.value: submit_rating, ROp.ADD_TAG.value: submit_tag }[op]
class FrontEnd: ''' Class for Front End Server within the distributed system. ''' def __init__(self): self.servers = [] self.rm = None # Initial selection of replica manager to communicate with try: self.rm = self._choose_replica() except ValueError as e: print(e) self.ts = VectorClock(REPLICA_NUM) # Vector timestamp of front end def send_request(self, request): ''' Method invoked by client to send a request. Params: (tuple) request: command to execute and arguments for the command Returns: If the request is a query, return the results of the query, otherwise a confirmation message. ''' r_type = self._request_type(request) # Find a replica manager to send request to if the original is # unavailable if self.rm is not None: try: rm_status = self.rm.get_status() print(rm_status) if rm_status == Status.OFFLINE.value: self.rm = self._choose_replica() except Pyro4.errors.ConnectionClosedError: self.rm = self._choose_replica() else: self.rm = self._choose_replica() if r_type == RType.UPDATE: rm_ts = self.rm.send_update(request, self.ts.value(), str(uuid.uuid4())) print('Update sent: ', request) self.ts.merge(VectorClock.fromiterable(rm_ts)) print('Front end timestamp: ', self.ts.value()) return 'Update submitted!' elif r_type == RType.QUERY: val, rm_ts = self.rm.send_query(request, self.ts.value()) print('Query sent: ', request) self.ts.merge(VectorClock.fromiterable(rm_ts)) print('Front end timestamp: ', self.ts.value()) return val def _choose_replica(self): ''' Select a replica manager to communicate with. Return: Remote object for a replica manager ''' for server in self.servers: server._pyroRelease() self.servers = self._find_replicas() stat = {server: server.get_status() for server in self.servers} available = [] num_offline = list(stat.values()).count(Status.OFFLINE.value) num_active = list(stat.values()).count(Status.ACTIVE.value) if num_active > 0: available = [ k for k in stat.keys() if stat[k] == Status.ACTIVE.value ] elif num_offline == len(self.servers): raise Exception('All servers offline') else: available = [ k for k in stat.keys() if stat[k] != Status.OFFLINE.value ] if not available: return None return random.choice(available) @staticmethod def _request_type(request): ''' Determine whether a request is an update or query. Params: (tuple) request: request to check Returns: Enum representing the type of request ''' op = request[0] op_type = op.split('.')[0] if op_type == 'u': return RType.UPDATE if op_type == 'q': return RType.QUERY raise ValueError('command not recognised') @staticmethod def _find_replicas(): ''' Find all online replica managers. Returns: servers: list of remote server objects for replica managers ''' servers = [] with Pyro4.locateNS() as ns: for server, uri in ns.list(prefix="network.replica.").items(): print("found replica", server) servers.append(Pyro4.Proxy(uri)) if not servers: raise ValueError( "No servers found! (are the movie servers running?)") return servers[:REPLICA_NUM]