class MetadataService(abstractMetadataService):
def __init__(self, nodes = ["node_1"]):
self._nodes = nodes
self._hr = HashRing(nodes=nodes)
self._key_to_node = {}
log.debug(f"Init of Metadata Service is complete. Nodes are: {nodes}")
def get_node(self, key):
hashed_key = self._hr.get_key(key)
log.debug(f"Retrieved hashed key: {hashed_key} from key: {key} from the hashring")
node = self._hr.get_node(hashed_key)
log.debug(f"The node to associated with the hashed key: {hashed_key} is: {node}")
self._key_to_node[key] = node
return node
def create_node(self, node_name):
self._hr.add_node(node_name)
self.nodes.append(node_name)
log.debug(f"Added the node: {node_name} to the hashring")
def get_all_metadata(self):
all_items = self._key_to_node.items()
log.debug(f"All the items in the metadata service are: {all_items}")
return all_items
def get_all_keys_for_node(self, node_name):
key_list = [key for key,node in self._nodes if node == node_name ]
log.debug(f"The mapping of keys to nodes in the metadata service is: {key_list}")
#TODO, allow reconstruction from a config file in case of power loss
def reconstruct_from_config(self, config):
pass
def test_ring_growth(ring):
add_ring = HashRing()
for nodename in ring.nodes:
add_ring.add_node(nodename)
assert ring.ring == add_ring.ring
assert ring.distribution == add_ring.distribution
class LRUCache():
'''
A least-recently used cache
* distributed, composed of server nodes (each server is a node)
* resize cluster dynamically
* each server node can have a custom MAX_SIZE and TIMEOUT
* data entries can expire if they are not called within TIMEOUT
* the LRU data entries are evicted first if MAX_SIZE is exceeded
* roughly even distribution of keys between server nodes
This implementation combines a hash ring with consistent hashing and a
doubly-linked list.
'''
def __init__(self, load_function):
'''
params:
load_function: on a cache miss, this function will be used to
load a value into the cache, given a key
'''
self.load_function = load_function
self.hr = HashRing(nodes=[])
self.servers = {}
def add_server(self, id, MAX_SIZE, TIMEOUT):
'''
Add a server to the ring.
params:
id: to identify the server
MAX_SIZE: int, max number of entries in the server
TIMEOUT: int or float, seconds after use before an entry times
out and is removed from the cache
'''
self.servers[id] = ServerNode(id, self.load_function, MAX_SIZE,
TIMEOUT)
self.hr.add_node(id)
def remove_server(self, id):
'''
Remove a server from the ring.
'''
del self.servers[id]
self.hr.remove_node(id)
def get(self, key):
'''
Return the value corresponding with a given key.
'''
target_server_id = self.hr.get_node(key)
return self.servers[target_server_id].get(key)
def get_state(self):
'''
Return dictionary in form {server_id: list of server nodes}
'''
server_contents = {}
if len(self.servers) > 0:
for server in self.servers:
server_contents[server] = self.servers[server].get_state()
return server_contents
def test_ring_growth_meta(ring_fast):
add_ring = HashRing(compat=False)
for nodename in ring_fast.nodes:
add_ring.add_node(nodename)
assert ring_fast._nodes == add_ring._nodes
assert ring_fast.ring == add_ring.ring
assert ring_fast.distribution == add_ring.distribution
def test_ring_growth_ketama(ring):
add_ring = HashRing(hash_fn='ketama')
for nodename in ring.nodes:
add_ring.add_node(nodename)
assert ring._nodes == add_ring._nodes
assert ring.ring == add_ring.ring
assert ring.distribution == add_ring.distribution
class ConsistentHashingRouter(Router):
("""Applies mapfunc to each message and based on the result, """
"""routes equal messages to always the same child""")
def __init__(self, name=None, mapfunc=None, *args, **kwargs):
self._hashring = HashRing()
self._map = mapfunc if callable(mapfunc) else lambda msg: msg
super().__init__(name=name, *args, **kwargs)
def _route(self, msg):
return self._hashring.get_node(self._map(msg))
def register_child(self, child):
super().register_child(child)
self._hashring.add_node(child)
def unregister_child(self, child):
super().unregister_child(child)
self._hashring.remove_node(child)
class LbDip(LBLogic):
def __init__(self, args):
super(LbDip, self).__init__(args)
self.nodes_names = [x for x in range(0, self.number_of_servers)]
self.collector = DipCollector(args)
self.hr = HashRing(nodes=self.nodes_names,
vnodes=args.vnodes) #TODO modify vnodes
self.connection_2_bucket = {}
self.past_connection = {
} # dictionay, used to record the bucket that each connection connected to, find connections are moved unnecessarily
self.server_2_buckets_connections = {
x: {}
for x in self.hr.get_nodes()
}
for bucket_id, node in self.hr.get_points():
self.server_2_buckets_connections[node][bucket_id] = []
def addNewConnection(self, packet):
bucket_id = self.hr.get_server(packet.getHeader())
serverID = bucket_id[1]
bucketID = bucket_id[0]
# bucket_id[1] is serverID, bucket_id[0] is the bucket
self.past_connection[packet.getHeader()] = [serverID]
self.collector.total_con += 1
self.collector.current_connection += 1
self.connection_2_bucket[
packet.getHeader()] = bucket_id # add bucket for that connection
self.server_2_buckets_connections[serverID][bucketID].append(
packet.getHeader()) # add connection on that bucket
self.collector.conclusion()
def removeConnection(self, packet):
bucket_id = self.connection_2_bucket[packet.getHeader()]
self.collector.current_connection -= 1
del self.connection_2_bucket[
packet.getHeader()] # remove bucket for that connection
del self.past_connection[packet.getHeader()]
self.server_2_buckets_connections[bucket_id[1]][bucket_id[0]].remove(
packet.getHeader()) #remove connection on that bucket
def addServer(self, serverId):
if serverId in self.nodes_names:
return
self.nodes_names.append(serverId)
all_connections = []
for D2value in self.server_2_buckets_connections.itervalues():
for D1value in D2value.itervalues():
all_connections += D1value
self.hr.add_node(serverId) # change the hash function
for each_connection in all_connections:
new_bucket_id = self.hr.get_server(each_connection)
old_bucket_id = self.connection_2_bucket[each_connection]
if str(old_bucket_id) != str(new_bucket_id):
if new_bucket_id[1] not in self.past_connection[
each_connection]:
self.past_connection[each_connection].append(
new_bucket_id[1])
else:
temp_index = self.past_connection[each_connection].index(
new_bucket_id[1])
self.past_connection[
each_connection] = self.past_connection[
each_connection][:temp_index + 1]
self.collector.unnecessary_move_count += 1
self.server_2_buckets_connections[
new_bucket_id[1]][new_bucket_id[0]].append(
each_connection) # add connection on that bucket
self.connection_2_bucket[
each_connection] = new_bucket_id # change bucket for that connection
self.server_2_buckets_connections[
old_bucket_id[1]][old_bucket_id[0]].remove(
each_connection) # delete connection on old bucket
def removeServer(self, serverId):
if serverId not in self.nodes_names:
return
self.nodes_names.remove(serverId)
buckets_to_be_removed = self.server_2_buckets_connections[serverId]
all_previous_connections = []
for bucket in self.server_2_buckets_connections[serverId]:
for each_connection in self.server_2_buckets_connections[serverId][
bucket]:
all_previous_connections.append(each_connection)
self.hr.remove_node(serverId)
for each_connection in all_previous_connections:
new_bucket = self.hr.get_server(each_connection)[0]
new_serverID = self.hr.get_server(each_connection)[1]
self.past_connection[each_connection].append(new_serverID)
self.server_2_buckets_connections[new_serverID][new_bucket].append(
each_connection)
self.connection_2_bucket[each_connection] = (new_bucket,
new_serverID)
for each_bucket in buckets_to_be_removed:
self.server_2_buckets_connections[serverId][each_bucket] = []
repSocket = context.socket(zmq.REP)
repSocket.bind("tcp://*:" + myPort)
reqSocket = context.socket(zmq.REQ)
mHashRing = HashRing(nodes=[myAddress])
mRingOrganizer = ringOrganizer(mHashRing, myRoPort)
mRingOrganizer.nodes.add(myAddress)
# if arguments are passed => this is a joining node
if len(sys.argv) > 1:
# get neighbor address
nPort = raw_input("neighbor port")
neighborAddress = '127.0.0.1:' + nPort
# add it to our own nodesTable
mRingOrganizer.nodes.add(neighborAddress)
mHashRing.add_node(neighborAddress)
# notify the neighbor to add us to it's table
msg = {'type': 'nodeJoinReq', 'address': myAddress}
#connect to neighborRingOrganizer
reqSocket.connect("tcp://" + getRoAddress(neighborAddress))
reqSocket.send(json.dumps(msg))
suggestedNodes = reqSocket.recv()
reqSocket.disconnect("tcp://" + getRoAddress(neighborAddress))
# TODO make this recursive
for nodeAddress in suggestedNodes.split(', '):
# if I dont know this node
if (nodeAddress not in mRingOrganizer.nodes):
mRingOrganizer.nodes.add(nodeAddress)
mHashRing.add_node(nodeAddress)
reqSocket.connect("tcp://" + getRoAddress(nodeAddress))
reqSocket.send(json.dumps(msg))
class MessageWorker(Thread):
def __init__(self, handler, server, living_list):
super(MessageWorker, self).__init__(target=self.run,
args=(),
daemon=True)
# 具体业务handler对象
self.handler = handler
# 指定服务点
self.server = server
# 接到最新通知的zk存活服务集合
self.living_list = living_list
# 使用中的服务存活状态
self.status_dict = {k: True for k in living_list}
# 应用中的hash环，注意每个worker应用的环并不是统一的，各点可能有自己的更新进度
self.ring = HashRing(living_list)
# 控制hash环状态锁
self.update_lock = Lock()
def get_server(self, message):
id = self.handler.get_message_id(message)
return self.ring.get_node(id)
# 启用新服务状态
def nodes_status_update(self):
if len(self.status_dict.keys() -
self.living_list) != 0 or len(self.living_list -
self.status_dict.keys()) != 0:
new_dict = {k: True for k in self.living_list}
new_list = list(
set(self.living_list) - set(self.status_dict.keys()))
gone_list = list(
set(self.status_dict.keys()) - set(self.living_list))
with self.update_lock:
self.status_dict = new_dict
for gone in gone_list:
self.ring.remove_node(gone)
for new in new_list:
self.ring.add_node(new)
dispatch_logger.info(
"[{}] [{}] [{}] enable new change.".format(
self.handler.name, self.server, self.__class__))
return True
# 接受广播函数，更新服务状态
def update_living_list(self, living_list):
with self.update_lock:
self.living_list = living_list
dispatch_logger.info("[{}] [{}] [{}] got nodes change.".format(
self.handler.name, self.server, self.__class__))
def get_server_is_living(self, server):
return self.status_dict.get(server, False)
def run(self):
pass
