class FrontEnd:
'''
Class for Front End Server within the distributed system.
'''
def __init__(self):
self.servers = []
self.rm = None
# Initial selection of replica manager to communicate with
try:
self.rm = self._choose_replica()
except ValueError as e:
print(e)
self.ts = VectorClock(REPLICA_NUM) # Vector timestamp of front end
def send_request(self, request):
'''
Method invoked by client to send a request.
Params:
(tuple) request: command to execute and arguments for the command
Returns:
If the request is a query, return the results of the query,
otherwise a confirmation message.
'''
r_type = self._request_type(request)
# Find a replica manager to send request to if the original is
# unavailable
if self.rm is not None:
try:
rm_status = self.rm.get_status()
print(rm_status)
if rm_status == Status.OFFLINE.value:
self.rm = self._choose_replica()
except Pyro4.errors.ConnectionClosedError:
self.rm = self._choose_replica()
else:
self.rm = self._choose_replica()
if r_type == RType.UPDATE:
rm_ts = self.rm.send_update(request, self.ts.value(),
str(uuid.uuid4()))
print('Update sent: ', request)
self.ts.merge(VectorClock.fromiterable(rm_ts))
print('Front end timestamp: ', self.ts.value())
return 'Update submitted!'
elif r_type == RType.QUERY:
val, rm_ts = self.rm.send_query(request, self.ts.value())
print('Query sent: ', request)
self.ts.merge(VectorClock.fromiterable(rm_ts))
print('Front end timestamp: ', self.ts.value())
return val
def _choose_replica(self):
'''
Select a replica manager to communicate with.
Return:
Remote object for a replica manager
'''
for server in self.servers:
server._pyroRelease()
self.servers = self._find_replicas()
stat = {server: server.get_status() for server in self.servers}
available = []
num_offline = list(stat.values()).count(Status.OFFLINE.value)
num_active = list(stat.values()).count(Status.ACTIVE.value)
if num_active > 0:
available = [
k for k in stat.keys() if stat[k] == Status.ACTIVE.value
]
elif num_offline == len(self.servers):
raise Exception('All servers offline')
else:
available = [
k for k in stat.keys() if stat[k] != Status.OFFLINE.value
]
if not available:
return None
return random.choice(available)
@staticmethod
def _request_type(request):
'''
Determine whether a request is an update or query.
Params:
(tuple) request: request to check
Returns:
Enum representing the type of request
'''
op = request[0]
op_type = op.split('.')[0]
if op_type == 'u':
return RType.UPDATE
if op_type == 'q':
return RType.QUERY
raise ValueError('command not recognised')
@staticmethod
def _find_replicas():
'''
Find all online replica managers.
Returns:
servers: list of remote server objects for replica managers
'''
servers = []
with Pyro4.locateNS() as ns:
for server, uri in ns.list(prefix="network.replica.").items():
print("found replica", server)
servers.append(Pyro4.Proxy(uri))
if not servers:
raise ValueError(
"No servers found! (are the movie servers running?)")
return servers[:REPLICA_NUM]
class ReplicaManager(threading.Thread):
'''
Class for a Replica Server within the distributed system, implementing
the gossip architecture.
'''
def __init__(self, replica_id, stopper, status=None):
super().__init__()
self._id = replica_id
# Replica status properties
self.failure_prob = 0.1
self.overload_prob = 0.2
self.auto_status = True
if status not in [n.value for n in list(Status)]:
print('Invalid status provided, defaulting to active.')
self.status = Status.ACTIVE
else:
self.status = Status(status)
self.auto_status = False
print(f'Status set to {status}.',
'Automatic status updating disabled.')
# Gossip Architecture State
self.value_ts = VectorClock(REPLICA_NUM) # aka data timestamp
self.replica_ts = VectorClock(REPLICA_NUM) # aka log timestamp
self.update_log = []
self.ts_table = [
VectorClock(REPLICA_NUM) if i != self._id else None
for i in range(REPLICA_NUM)
]
self.executed = []
self.pending_queries = queue.Queue()
self.query_results = {}
self.interval = 8.0 # interval between gossip exchanges
self.other_replicas = self._find_replicas()
self.stopper = stopper # Used to indicate to server to stop
# Locks for objects shared between threads
self.vts_lock = threading.Lock() # for value_ts
self.rts_lock = threading.Lock() # for replica_ts
self.log_lock = threading.Lock() # for update_log
def run(self):
'''
Override of threading.Thread run() method. Sends gossip to other
replica managers periodically.
'''
while not self.stopper.is_set():
if self.status != Status.OFFLINE:
for r_id, rm in self.other_replicas:
rm._pyroRelease()
self.other_replicas = self._find_replicas()
with self.rts_lock:
print('\n--- SENDING GOSSIP ---')
for r_id, rm in self.other_replicas:
r_ts = self.ts_table[r_id]
m_log = self._get_recent_updates(r_ts)
print(f'Updates to send to RM {r_id}: ', m_log)
try:
rm.send_gossip(m_log, self.replica_ts.value(),
self._id)
print(f'Gossip sent to RM {r_id}')
except Pyro4.errors.CommunicationError as e:
print(f'Failed to send gossip to RM {r_id}')
print('----------------------')
if self.auto_status:
self._update_status()
print('Status: ', self.status.value, '\n')
self.stopper.wait(self.interval)
print('Stopper set, gossip thread stopping.')
def send_query(self, q_op, q_prev):
'''
Method invoked by the front end to send a query.
Params:
(string) q_op: query command
(tuple) q_prev: vector timestamp of front end
Returns:
response: results of query
'''
print('Query received: ', q_op, q_prev)
response = None
q_prev = VectorClock.fromiterable(q_prev)
# stable = are we up to date enough to handle the query correctly?
stable = False
with self.vts_lock:
if q_prev <= self.value_ts: # stability criteria for query
val = self._apply_query(q_op)
new = self.value_ts.value()
response = (val, new)
stable = True
print('Value timestamp: ', self.value_ts.value(), '\n')
if not stable:
# if not stable, add to a dictionary of pending queries and wait
self.query_results[(q_op, q_prev.value())] = queue.Queue(maxsize=1)
self.pending_queries.put((q_op, q_prev))
# Wait for query to be executed after some gossip exchange
response = self.query_results[(q_op, q_prev.value())].get()
# Remove entry from pending query dictionary
del self.query_results[(q_op, q_prev.value())]
return response
def send_update(self, u_op, u_prev, u_id):
'''
Method invoked by the front end to send an update.
Params:
(string) u_op: update command
(tuple) u_prev: vector timestamp of front end
(string) u_id: unique ID for update
Returns:
ts: timestamp representing having executed the update or None
if the update has already been executed
'''
print('Update received: ', u_op, u_prev, u_id)
ts = None
# Add update to log if it hasn't already been executed
if u_id not in self.executed:
with self.rts_lock:
self.replica_ts.increment(self._id)
ts = list(u_prev[:])
ts[self._id] = self.replica_ts.value()[self._id]
print('Replica timestamp: ', self.replica_ts, '\n')
ts = VectorClock.fromiterable(ts)
u_prev = VectorClock.fromiterable(u_prev)
log_record = (self._id, ts, u_op, u_prev, u_id)
with self.log_lock:
self.update_log.append(log_record)
print('Update record: ', log_record)
# Execute update if it is stable
with self.vts_lock:
if u_prev <= self.value_ts: # stability criteria for query
self._execute_update(u_op, u_id, ts)
return ts.value()
return ts
@Pyro4.oneway
def send_gossip(self, m_log, m_ts, r_id):
'''
Method invoked by other replica managers to send gossip.
Params:
(string) m_log: recent updates from replica manager
(tuple) m_ts: log timestamp of sending replica manager
(string) r_id: ID of sending replica manager
Returns:
ts: timestamp representing having executed the update or None
if the update has already been executed
'''
if self.status != Status.OFFLINE:
print('\n--- RECEIVING GOSSIP ---')
print(f'Gossip received from RM {r_id}')
print(m_ts)
print(m_log)
print()
# Merge m_log into update log
self._merge_update_log(m_log)
# Merge our replica timestamp with m_ts
m_ts = VectorClock.fromiterable(m_ts)
with self.rts_lock:
self.replica_ts.merge(m_ts)
print('Replica timestamp: ', self.replica_ts)
# Execute all updates that have now become stable
stable = self._get_stable_updates()
for update in stable:
_id, ts, u_op, u_prev, u_id = update
with self.vts_lock:
self._execute_update(u_op, u_id, ts)
# Set the timestamp of the sending replica manager in our timestamp
# table
self.ts_table[r_id] = m_ts
# Execute all stable pending queries
while True:
try:
q_op, q_prev = self.pending_queries.get(block=False)
with self.vts_lock:
if q_prev <= self.value_ts:
val = self._apply_query(q_op)
new = self.value_ts.value()
self.query_results[(q_op, q_prev.value())].put(
(val, new))
except queue.Empty:
break
print('------------------------')
def get_status(self):
'''
Method invoked by front end to query the server status.
Returns:
status of the server
'''
return self.status.value
def set_status(self, status):
'''
Method invoked by status_control.py to set the server status.
'''
self.status = Status(status)
def toggle_auto_status(self, auto):
'''
Method invoked by status_control.py to set the server status to
update automatically or not.
'''
if auto:
self.auto_status = True
else:
self.auto_status = False
def _update_status(self):
'''
Set the server status probabilistically.
'''
overloaded = random.random()
failed = random.random()
if failed < self.failure_prob:
self.status = Status.OFFLINE
elif overloaded < self.overload_prob:
self.status = Status.OVERLOADED
else:
self.status = Status.ACTIVE
def _apply_query(self, q_op):
'''
Execute a query command.
Params:
(string) q_op: query command to execute
Returns:
val: result of query
'''
print('Query applied. ', q_op, '\n')
val = None
op, *params = q_op
query = self._parse_q_op(op)
val = query(*params)
return val
def _apply_update(self, u_op):
'''
Execute an update command.
Params:
(string) u_op: update command to execute
'''
print('Update applied.', u_op, '\n')
op, *params = u_op
update = self._parse_u_op(op)
update(*params)
def _execute_update(self, u_op, u_id, ts):
'''
Execute an update.
Params:
(string) u_op: update command to execute
(string) u_id: ID of update to execute
(VectorClock) ts: timestamp of update to execute
'''
# Return immediately if update has already been executed
if u_id in self.executed:
return
self._apply_update(u_op) # Execute the update
self.value_ts.merge(ts) # Update the value timestamp
self.executed.append(u_id) # Add update to executed updates
print('Value timestamp: ', self.value_ts)
def _merge_update_log(self, m_log):
'''
Merge the update log with updates from a gossip message.
Params:
m_log: list of updates from a gossip message
'''
for record in m_log:
_id, ts, u_op, u_prev, u_id = record
ts = VectorClock.fromiterable(ts)
u_prev = VectorClock.fromiterable(u_prev)
with self.rts_lock, self.log_lock:
new_record = (_id, ts, u_op, u_prev, u_id)
if new_record not in self.update_log:
if not ts <= self.replica_ts:
self.update_log.append(new_record)
def _get_stable_updates(self):
'''
Retrieve all stable updates from the update log.
Returns:
stable: list of updates that can be executed.
'''
stable = []
with self.vts_lock, self.log_lock:
stable = [
record for record in self.update_log
if record[3] <= self.value_ts
]
stable.sort(key=lambda r: r[3])
return stable
def _get_recent_updates(self, r_ts):
'''
Retrieve updates from update log that are more recent than our recorded
value of the timestamp of another replica manager.
Params:
(VectorClock) r_ts: Timestamp of another replica manager, sent in
gossip
Returns:
recent: all updates from update log that are more recent than the
given timestamp
'''
recent = []
with self.log_lock:
for record in self.update_log:
_id, ts, u_op, u_prev, u_id = record
if ts > r_ts:
new_record = (_id, ts.value(), u_op, u_prev.value(), u_id)
recent.append(new_record)
return recent
def _find_replicas(self):
'''
Find all online replica managers.
Returns:
servers: list of remote server objects for replica managers
'''
servers = []
try:
with Pyro4.locateNS() as ns:
for server, uri in ns.list(prefix="network.replica.").items():
server_id = int(server.split('.')[-1])
if server_id != self._id:
servers.append((server_id, Pyro4.Proxy(uri)))
except Pyro4.errors.NamingError:
print('Could not find Pyro nameserver.')
servers.sort()
return servers[:REPLICA_NUM]
@staticmethod
def _parse_q_op(op):
'''
Match query command strings with query functions.
Params:
(string) op: query command
Returns:
function corresponding to the query command
'''
return {
ROp.GET_AVG_RATING.value: get_avg_movie_rating,
ROp.GET_RATINGS.value: get_movie_ratings,
ROp.GET_GENRES.value: get_movie_genres,
ROp.GET_MOVIE.value: get_movie_by_title,
ROp.GET_TAGS.value: get_movie_tags,
ROp.SEARCH_TITLE.value: search_by_title,
ROp.SEARCH_GENRE.value: search_by_genre,
ROp.SEARCH_TAG.value: search_by_tag
}[op]
@staticmethod
def _parse_u_op(op):
'''
Match update command strings with update functions.
Params:
(string) op: update command
Returns:
function corresponding to the update command
'''
return {
ROp.ADD_RATING.value: submit_rating,
ROp.ADD_TAG.value: submit_tag
}[op]
class Node(KV_store):
'''docstring for node class'''
def __init__(self, router, address, view, replication_factor):
self.gossiping = False
self.sched = Scheduler()
self.sched.start()
KV_store.__init__(self, address)
self.history = [('Initialized', datetime.now())]
self.ADDRESS = address
self.VC = VectorClock(view=view, clock=None)
self.ring_edge = 691 if len(
view) < 100 else 4127 # parameter for hash mod value
self.repl_factor = replication_factor
self.num_shards = 0
self.virtual_range = 10
self.shard_interval = self.ring_edge // self.virtual_range
self.nodes = []
self.shard_ID = -1
self.V_SHARDS = [] # store all virtual shards
self.P_SHARDS = [[] for i in range(0, self.num_shards)
] # map physical shards to nodes
self.virtual_translation = {} # map virtual shards to physical shards
self.backoff_mod = 113
self.router = router
self.view_change(view, replication_factor)
def __repr__(self):
return {
'ADDRESS': self.ADDRESS,
'V_SHARDS': self.V_SHARDS,
'P_SHARDS': self.P_SHARDS,
'KEYS': len(self.keystore)
}
def __str__(self):
return 'ADDRESS: ' + self.ADDRESS + '\nREPL_F: ' + str(
self.repl_factor) + '\nNODES: ' + (', '.join(map(
str, self.nodes))) + '\nP_SHARDS: ' + (', '.join(
map(str, self.P_SHARDS)))
'''
give a state report
this includes node data and distribution of keys to nodes
'''
def state_report(self):
state = self.__repr__()
state['HISTORY'] = {}
string = 'node'
itr = 1
for event in self.history:
key = string + str(itr)
itr += 1
state['HISTORY'][key] = event
return state
'''
return all physical shards
'''
def all_shards(self):
return self.P_SHARDS
def all_nodes(self):
return self.nodes
'''
get all nodes in this shard
'''
def shard_replicas(self, shard_ID):
return self.P_SHARDS[shard_ID]
'''
hash frunction is a composit of xxhash modded by prime
'''
def hash(self, key, Type):
hash_val = hasher.xxh32(key).intdigest()
# may be expensive but will produce better distribution
return (hash_val % self.ring_edge)
'''
evenly distribute nodes into num_shard buckets
'''
def even_distribution(self, repl_factor, nodes):
nodes.sort()
num_shards = (len(nodes) // repl_factor)
replicas = (len(nodes) // num_shards)
overflow = (len(nodes) % num_shards)
shards = [[] for i in range(0, num_shards)]
shard_dict = {}
node_iter = 0
for shard in range(num_shards):
extra = (1 if shard < overflow else 0)
interval = replicas + extra
shards[shard] = nodes[node_iter:(node_iter + interval)]
node_iter += interval
for node in shards[shard]:
shard_dict[node] = shard
return shard_dict
'''
Perform a key operation, ie. find the correct shard given key.
First hash the key then perform binary search to find the correct shard
to store the key.
'''
def find_match(self, key):
ring_val = self.hash(key, 'consistent')
# get the virtual shard number
v_shard = self.find_shard('predecessor', ring_val)
# convert to physical shard
shard_ID = self.virtual_translation[v_shard]
return shard_ID
'''
perform binary search on list of virtual shards given ring value
we need to be careful about wrap around case. If ring_val >= max_ring_val, return 0
'''
def find_shard(self, direction, ring_val):
if direction == 'predecessor':
v_shard = bisect_left(self.V_SHARDS, ring_val)
if v_shard:
return self.V_SHARDS[v_shard - 1]
return self.V_SHARDS[-1]
elif direction == 'successor':
v_shard = bisect_right(self.V_SHARDS, ring_val)
if v_shard != len(self.V_SHARDS):
return self.V_SHARDS[v_shard]
return self.V_SHARDS[0]
'''
respond to view change request, perform a reshard
this can only be done if all nodes have been given new view
2 cases:
1. len(nodes) + 1 // r > or < shard_num: we need to add or
remove a shard to maintain repl_factor
2. add and/or remove nodes
'''
def view_change(self, view, repl_factor):
new_num_shards = len(view) // repl_factor
if new_num_shards == 1:
new_num_shards = 2
view.sort()
buckets = self.even_distribution(repl_factor, view)
#print('buckets', buckets)
# add nodes and shards
for node in view:
my_shard = buckets[node]
if node == self.ADDRESS:
self.shard_ID = buckets[node]
self.sched.add_interval_job(self.gossip,
seconds=self.gossip_backoff())
# add a new node
if node not in self.nodes:
self.add_node(node, my_shard, new_num_shards)
# move node to new shard
else:
if my_shard >= len(self.P_SHARDS):
self.add_shard()
if node not in self.P_SHARDS[my_shard]:
self.move_node(node, my_shard)
old_nodes = list(set(self.nodes) - set(view))
# remove nodes from view
for node in old_nodes:
self.remove_node(node)
# remove empty shards
for shard_ID in range(0, len(self.P_SHARDS)):
if len(self.P_SHARDS[shard_ID]) == 0:
self.remove_shard(shard_ID)
'''
Add a single node to shards and get keys from shard replicas
'''
def add_node(self, node, shard_ID, num_shards):
# do we need to add another shard before adding nodes
while num_shards > self.num_shards:
self.add_shard()
# update internal data structures
self.nodes.append(node)
self.nodes.sort()
self.P_SHARDS[shard_ID].append(node)
# determine if the node's shard is this shard
if self.shard_ID == shard_ID:
#print('<adding node to:', shard_ID)
self.shard_keys()
'''
move node from old shard to new shard and perform atomic key transfer
'''
def move_node(self, node, shard_ID):
old_shard_ID = self.nodes.index(node) // self.num_shards
if node not in self.P_SHARDS[old_shard_ID]:
if old_shard_ID > 0 and node in self.P_SHARDS[old_shard_ID - 1]:
old_shard_ID += -1
else:
old_shard_ID += 1
# do we need to add another shard before adding nodes
while shard_ID > len(self.P_SHARDS):
self.add_shard()
self.atomic_key_transfer(old_shard_ID, shard_ID, node)
self.P_SHARDS[shard_ID].append(node)
self.P_SHARDS[old_shard_ID].pop(
self.P_SHARDS[old_shard_ID].index(node))
'''
remove single node from a shard and send final state to shard replicas
'''
def remove_node(self, node):
shard_ID = (self.nodes.index(node) - 1) // self.num_shards
if shard_ID > 0 and shard_ID < len(
self.P_SHARDS) and node not in self.P_SHARDS[shard_ID]:
if shard_ID > 0 and node in self.P_SHARDS[shard_ID - 1]:
shard_ID += -1
else:
shard_ID += 1
#print('error finding node')
if node == self.ADDRESS:
print('<send my final state to my replicas before removing')
success = self.final_state_transfer(node)
if success:
self.nodes.pop(self.nodes.index(node))
else:
raise Exception('<final_state_transfer failed>')
else:
self.nodes.pop(self.nodes.index(node))
self.P_SHARDS[shard_ID].pop(self.P_SHARDS[shard_ID].index(node))
'''
add shard to view
'''
def add_shard(self):
new_shards = []
p_shard = self.num_shards
if p_shard >= len(self.P_SHARDS):
self.P_SHARDS.append([])
for v_shard in range(self.virtual_range):
virtural_shard = str(p_shard) + str(v_shard)
ring_num = self.hash(virtural_shard,
'consistent') # unique value on 'ring'
# if ring_num is already in unsorted list, skip this iteration
if ring_num in self.V_SHARDS:
#print('<System: Hash collision detected>')
continue
self.V_SHARDS.append(ring_num)
self.virtual_translation[ring_num] = p_shard
self.num_shards += 1
self.V_SHARDS.sort()
return new_shards
'''
remove from all internal data structures if there are no nodes in shard
'''
def remove_shard(self, shard_ID):
self.P_SHARDS.pop(shard_ID)
'''
get all keys for a given shard
'''
def shard_keys(self):
pass
'''
perform an atomic key transfer
concurrent operation: get new keys, send old keys, delete old keys
'''
def atomic_key_transfer(self, old_shard_ID, new_shard_ID, node):
# message all nodes and tell them your state
# get new keys from new replica
self.final_state_transfer()
old_kv = self.KV_store
for replica in self.P_SHARDS[old_shard_ID]:
data = None
try:
res, status_code = self.router.GET(replica,
'/kv-store/internal/KV',
data, False)
except:
continue
if status_code == 201:
new_kv = res.get('KV_store')
update = False
for key in new_kv:
self.KV_store.keystore[key] = new_kv[key]
for key in old_kv:
del self.KV_store.keystore[key]
return True
return False
'''
send final state of node before removing a node
'''
def final_state_transfer(self, node):
data = {"kv-store": self.keystore, "context": self.VC.__repr__()}
replica_ip_addresses = self.shard_replicas(self.shard_ID)
for replica in replica_ip_addresses:
if (replica != self.ADDRESS):
try:
res, status_code = self.router.PUT(
replica, '/kv-store/internal/state-transfer', data,
False)
except:
continue
if status_code == 201:
return True
return False
'''
handle node failures, check if node should be removed or not
'''
def handle_unresponsive_node(self, node):
pass
def gossip_backoff(self):
return hash(self.ADDRESS) % random.randint(20, 40)
def gossip(self):
if (self.gossiping == False):
current_key_store = self.keystore
self.gossiping = True
replica_ip_addresses = self.shard_replicas(self.shard_ID)
replica = replica_ip_addresses[(random.randint(
0,
len(replica_ip_addresses) - 1))]
while (self.ADDRESS == replica):
replica = replica_ip_addresses[(random.randint(
0,
len(replica_ip_addresses) - 1))]
myNumber = int((self.ADDRESS.split(".")[3]).split(":")[0])
otherNumber = int((replica.split(".")[3]).split(":")[0])
tiebreaker = replica if (otherNumber > myNumber) else self.ADDRESS
data = {
"context": self.VC.__repr__(),
"kv-store": current_key_store,
"tiebreaker": tiebreaker
}
print("sending to node: " + replica + " " + str(data),
file=sys.stderr)
try:
response = self.router.PUT(replica,
'/kv-store/internal/gossip/',
json.dumps(data))
except:
code = -1
code = response.status_code
if (code == 200):
# 200: They took my data
self.gossiping = False
elif (code == 501):
content = response.json()
# 501:
# the other node was either the tiebreaker or happened after self
# so this node takes its data
# context of node
other_context = content["context"]
# key store of incoming node trying to gossip
other_kvstore = content["kv-store"]
incoming_Vc = VectorClock(view=None, clock=other_context)
if bool(other_kvstore) and not incoming_Vc.allFieldsZero():
if current_key_store == self.keystore:
print("I TOOK DATA: " + str(self.keystore),
file=sys.stderr)
self.VC.merge(other_context, self.ADDRESS)
self.keystore = other_kvstore
else:
print("I RECIEVED AN UPDATE WHILE GOSSIPING, ABORT",
file=sys.stderr)
self.gossip = False
#self happened before other, take its kvstore and merge with my clock
# concurrent but other is tiebreaker
else:
# 400: Other is already gossiping with someone else
# ELSE: unresponsive node (maybe itll be code 404?)
self.gossiping = False
else:
# Curretly gossiping,
# Will call after gossip backoff again
self.gossiping = False
return 200
