def __init__(self, join_ip, topic="topic"): self.listID = [] for i in range(0, 256): self.listID.append(i) ring = HashRing(self.listID) self.id = ring.get_node(topic) print "id assigned:", self.id self.join_ip = join_ip self.topic = topic self.context = zmq.Context() s, self.event_service_ip = self.call_remote_procedure( self.join_ip, "findSuccessor", str(self.id)).split() print "EventService ip: ", self.event_service_ip self.event_service_id = int(s) xsub_url = "tcp://" + self.event_service_ip + ":5556" socket = self.context.socket(zmq.PUB) socket.connect(xsub_url) # pub.bind(xpub_url) for n in range(1000): priority = randint(0, 9) msg = [topic, str(priority), str(n)] socket.send_multipart(msg) print "for the message %s with priority %d" % ( topic, priority), "process time in publisher is", time.clock() time.sleep(1.25)
def __init__(self, addr, port): RPCServer.__init__(self, addr, port) len_up = len(UPLOAD_SERVERS) len_repo = len(REPOSITORY_SERVERS) if len_up < len_repo or len_up % len_repo != 0: show_error(self, 'failed to initialize') raise Exception('failed to initialize') addr = localhost() if addr not in REPOSITORY_SERVERS: show_error(self, 'failed to initialize') raise Exception('failed to initialize REPOSITORY_SERVERS') for i in range(len(REPOSITORY_SERVERS)): if addr == REPOSITORY_SERVERS[i]: break total = len_up / len_repo self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total] self._print('upload_servers=%s' % str(self._upload_servers)) if HDFS: self._port = HDFS_PORT self._client = HDFSClient() else: self._port = FTP_PORT self._client = FTPClient() self._server = FTPServer() if REPO_DB: self._db = Database(addr=REPO_DB) else: self._db = Database(addr=addr) locks = [] for _ in range(LOCK_MAX): locks.append(Lock()) self._locks = HashRing(locks) if DEBUG: self._upload_cnt = 0 self._download_cnt = 0
def ring_fast(): r = HashRing() nodes = [] for i in range(num_nodes): nodes.append(HashRingNode(b'test-%i' % i, num_replicas=512)) r.add_nodes(nodes)
def __init__(self, hosts, connect=True): """ @param hosts: A C{list} of C{tuple}s containing hosts and ports. """ self.hosts = hosts self.hash_ring = HashRing(hosts) if connect: self.connect()
def getClient(self, key): hosts = self.getActiveConnections() if hosts != self.hash_ring.nodes: self.hash_ring = HashRing(hosts) log.msg("Using %i active hosts" % len(hosts)) if len(hosts) == 0: raise NoServerError("No connected servers remaining.") return self.hash_ring.get_node(key)
def __init__(self, servers, *k, **kw): self.hash_ring = HashRing(servers) memcache.Client.__init__(self, servers, *k, **kw) self.server_mapping = {} for server_uri, server_obj in zip(servers, self.servers): self.server_mapping[server_uri] = server_obj
def __init__(self, nodes): """构造函数 nodes 节点数据,格式:{'node01': resource01, 'node02': resource02} """ self._nodes = nodes ids = nodes.keys() ids.sort() self._ring = HashRing(ids)
def __init__(self, location=None, base_url=None, **kwargs): self.clients = {} for server in settings.DISTRIBUTED_MOGILEFS_CONFIG['SERVERS']: srv = settings.DISTRIBUTED_MOGILEFS_CONFIG['SERVERS'][server] self.clients[server] = Client(domain=srv['DOMAIN'], trackers=srv['TRACKERS']) self.servers = settings.DISTRIBUTED_MOGILEFS_CONFIG['SERVERS'] self.ring = HashRing(settings.DISTRIBUTED_MOGILEFS_CONFIG['SLOTS']) self.kwargs = kwargs
def __init__(self, hosts, timeout=5, lazy=False): self._hosts = hosts self._timeout = timeout self._sockets = None self._ring = HashRing(self._hosts) if not lazy: self.open()
def create_sets(servers): server_sets = {} for s in servers: server_sets[s] = set() ring = HashRing(servers) for word in palindromes: node = ring.get_node(word) server_sets[node].add(word) return server_sets
def test_iterate_nodes(): simple_list = ['1', '2', '3', '4', '5'] new_ring = HashRing(simple_list) nodes = [] for node in new_ring.iterate_nodes('a'): nodes.append(node) assert len(nodes) == len(simple_list) for elm in simple_list: assert elm in nodes
def take_over(self, req): """Attached to ``POST /take-over`` Takes over databases from another server, that presumably has gone offline without notice. This goes through all of the local databases, and sees if this node was either using the bad node as a backup, or is a backup for the bad node. In either case it finds the new node that should be either master or handling the bad node, and sends the local database to that server. Takes a JSON body with keys: `other`: a list of all nodes `name`: the name of *this* node `bad`: the bad node being removed `backups`: the number of backups """ self.assert_is_internal(req) status = Response(content_type='text/plain') data = req.json nodes = data['other'] self_name = data['name'] bad_node = data['bad'] assert self_name != bad_node backups = data['backups'] ring = HashRing(nodes) for domain, username, bucket in self.storage.all_dbs(): assert bucket.startswith('/') path = '/' + domain + '/' + username + bucket iterator = iter(ring.iterate_nodes(path)) active_nodes = [iterator.next() for i in xrange(backups + 1)] replacement_node = iterator.next() # Not all the backups should try to restore the database, so instead # just the "first" does it restore = False if active_nodes[0] == bad_node and active_nodes[1:] and active_nodes[1] == self_name: status.write('Master node %s for %s removed\n' % (bad_node, path)) restore = True elif bad_node in active_nodes and active_nodes[0] == self_name: status.write('Backup node %s for %s removed\n' % (bad_node, path)) restore = True if not restore: continue db = self.storage.for_user(domain, username, bucket) send = Request.blank(replacement_node + urllib.quote(path) + '?paste', method='POST', body=''.join(db.encode_db())) send.environ['cutout.root'] = req.environ.get('cutout.root') resp = forward(send) assert resp.status_code == 201, str(resp) #status.write(' nodes: %r - %r / %r\n' % (active_nodes, bad_node, self_name)) status.write(' success, added to %s (from %s)\n' % (replacement_node, self_name)) return status
class LocatorHandler(BaseHandler, Locator.Iface): def __init__(self, peer=None, port=9900): self.address = socket.gethostbyname(socket.gethostname()) self.port = port self.peer = peer self.ring = HashRing() try: ping(self.location) print 'Uh-oh. Our location responded to a ping!' raise socket.error(43, 'Address already in use') except NodeNotFound: pass @property def here(self): "Give the canonical string representation" return loc2str(self) @property def location(self): "Give the canonical Location" return Location(address=self.address, port=self.port) def join(self, location): """ Parameters: - location """ self.add(location, [self.location]) ping_until_return(location) items = self.ring.nodes.difference([loc2str(location)]) def remove(self, location, authorities): """ Parameters: - location - authorities """ key = loc2str(location) self.ring.remove(loc2str(location)) authorities.append(self.location) destinations = select_peers(self.ring.nodes.difference(map(loc2str,authorities))) for destination in destinations: try: remote_call('remove', str2loc(destination), location, authorities) break except NodeNotFound, tx: # enter all nodes as authorities to avoid race conditions # lazy invalidation self.remove(tx.location, map(str2loc, self.ring.nodes)) print "removed %s:%d" % (location.address, location.port)
def url(self, name, node=None, **kwargs): name = self.path(name) if not name: return "" node_key = node or self.get_key_by_name(name, **kwargs) name = name.lstrip("/") urls = self.servers[node_key]["URLS"] urls = [urls] if isinstance(urls, basestring) else urls if len(urls) > 1: ring2 = HashRing(urls) base_url = ring2.get_node(node_key) else: base_url = urls[0] return base_url + name
def __init__(self, addr, port): RPCServer.__init__(self, addr, port, user=User()) self._sandbox = Sandbox() locks = [] for _ in range(LOCK_MAX): locks.append(Lock()) self._locks = HashRing(locks) self._cache = {} if DEBUG: self._register_cnt = 0 self._login_cnt = 0 self._upload_cnt = 0 self._install_cnt = 0 self._uninstall_cnt = 0
def url(self, name, node=None, **kwargs): name = self.path(name) if not name: return '' node_key = node or self.get_key_by_name(name, **kwargs) name = name.lstrip('/') urls = self.servers[node_key]['URLS'] urls = [urls] if isinstance(urls, basestring) else urls if len(urls) > 1: ring2 = HashRing(urls) base_url = ring2.get_node(node_key) else: base_url = urls[0] return base_url + name
def test_with_objects(): simple_list = [Server(1), Server(2), Server(3)] new_ring = HashRing(simple_list) node = new_ring.get_node('BABU') assert node in simple_list nodes = [] for node in new_ring.iterate_nodes('aloha'): nodes.append(node) assert len(nodes) == len(simple_list) for elm in simple_list: assert elm in nodes
class MemcacheRing(memcache.Client): """Extends python-memcache so it uses consistent hashing to distribute the keys. """ def __init__(self, servers, *k, **kw): self.hash_ring = HashRing(servers) memcache.Client.__init__(self, servers, *k, **kw) self.server_mapping = {} for server_uri, server_obj in zip(servers, self.servers): self.server_mapping[server_uri] = server_obj def _get_server(self, key): if isinstance(key, tuple): return memcache.Client._get_server(key) for i in range(self._SERVER_RETRIES): iterator = self.hash_ring.iterate_nodes(key) for server_uri in iterator: server_obj = self.server_mapping[server_uri] if server_obj.connect(): return server_obj, key return None, None
class MemcacheRing(memcache.Client): """Extends python-memcache so it uses consistent hashing to distribute the keys. """ def __init__(self, servers, *k, **kw): self.hash_ring = HashRing(servers) memcache.Client.__init__(self, servers, *k, **kw) self.server_mapping = {} for server_uri, server_obj in zip(servers, self.servers): self.server_mapping[server_uri] = server_obj def _get_server(self, key): if type(key) == types.TupleType: return memcache.Client._get_server(key) for i in range(self._SERVER_RETRIES): iterator = self.hash_ring.iterate_nodes(key) for server_uri in iterator: server_obj = self.server_mapping[server_uri] if server_obj.connect(): return server_obj, key return None, None
def __init__(self, query): Thread.__init__(self) self._pool = ThreadPool(processes=4) self._workers = HashRing(WORKER_SERVERS) self._identity = bytes(uuid.uuid4()) self._query = query self._init_sock()
def receive_slave_failure(self): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) host = socket.gethostname() print 'receiving failure' sock.bind((self.host, 10012)) global backup_slave_nodes while True: request, addr = sock.recvfrom(1024) #request will be just the slave number dead_slaves = json.loads(request) print 'Recieved dead slaves : ' print dead_slaves for slave in dead_slaves: if slave in self.slave_nodes.keys(): endpoint = self.slave_nodes[slave] del self.slave_nodes[slave] del self.backup[slave] self.memcache_servers.remove(endpoint) print 'after deletion'+str(self.slave_nodes) #Now add the backup slave nodes self.slave_nodes[slave]=backup_slave_nodes[slave] self.backup[slave]=backup_slave_nodes[slave] self.memcache_servers.append(backup_slave_nodes[slave]) print 'after addition'+str(self.slave_nodes) #change the ring using the new slave self.ring = HashRing(self.memcache_servers)
def __init__(self, conns, pipelines=None): length = len(conns) if length not in ShardRedis.cache: ShardRedis.cache[length] = HashRing(range(length)) self.ring = ShardRedis.cache[length] self.conns = conns self.pipelines = pipelines
def __init__(self,portNumber): self.portNumber = portNumber self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.host = socket.gethostname() print self.host self.sock.bind((self.host, self.portNumber)) self.monitors = dict() with open("config/monitor.txt") as myfile: for line in myfile: name, endpoint = line.partition("=")[::2] print endpoint self.monitors[name] = endpoint self.pool = ThreadPool(10) #TODO : configure this self.memcache_servers = [] self.slave_nodes = dict() self.backup = dict() self.config = {"result":"New","host":self.host,"port":self.portNumber} sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) host,port = self.monitors[self.monitors.keys()[0]].split(":") sock.sendto(json.dumps(self.config), (host,int(port))) slave_config,addr = self.sock.recvfrom(1024) sock.close() with open("config/slave.txt",'w') as fin: for val in slave_config: fin.write(str(val)) with open("config/slave.txt",'r') as myfile: for line in myfile: name, endpoints = line.partition("=")[::2] endpoint1,endpoint2= endpoints.split(',') self.memcache_servers.append(endpoint1) self.slave_nodes[name]=endpoint1 self.backup[name] =endpoint2 self.ring = HashRing(self.memcache_servers) print self.backup
def query_deprecate(self, req): """Responds to ``POST /query-deprecate`` This is used when a new node is added to the system, and all existing nodes are asked for what databases should be assigned to the new node. Any such database will be deprecated, and a list of those databases is returned. Accepts a JSON body with the keys: `other`: list of all nodes `name`: the name of this node `new`: the node being added `backups`: the number of backups to keep Returns JSON:: {"deprecated": [deprecated items]} Where the deprecated items are:: {"path": "/domain/user/bucket", "domain": "domain", "username": "******", "bucket": "bucket" } """ self.assert_is_internal(req) data = req.json nodes = data['other'] self_name = data['name'] new_node = data['new'] backups = data['backups'] ring = HashRing(nodes + [new_node]) deprecated = [] for domain, username, bucket in self.storage.all_dbs(): assert bucket.startswith('/') path = '/' + domain + '/' + username + bucket iterator = iter(ring.iterate_nodes(path)) active_nodes = [iterator.next() for i in xrange(backups + 1)] deprecated_node = iterator.next() if deprecated_node == self_name and new_node in active_nodes: deprecated.append( {'path': path, 'domain': domain, 'username': username, 'bucket': bucket}) db = self.storage.for_user(domain, username, bucket) db.deprecate() return Response(json={'deprecated': deprecated})
def __init__(self, location=None, base_url=None, **kwargs): self.clients = {} for server in settings.DISTRIBUTED_MOGILEFS_CONFIG["SERVERS"]: srv = settings.DISTRIBUTED_MOGILEFS_CONFIG["SERVERS"][server] self.clients[server] = Client(domain=srv["DOMAIN"], trackers=srv["TRACKERS"]) self.servers = settings.DISTRIBUTED_MOGILEFS_CONFIG["SERVERS"] self.ring = HashRing(settings.DISTRIBUTED_MOGILEFS_CONFIG["SLOTS"]) self.kwargs = kwargs
def __init__(self, join_ip, topic="topic"): self.listID = [] for i in range(0, 256): self.listID.append(i) ring = HashRing(self.listID) self.id = ring.get_node(topic) print "id assigned:", self.id self.join_ip = join_ip self.topic = topic self.context = zmq.Context() s, self.event_service_ip = self.call_remote_procedure( self.join_ip, "findSuccessor", str(self.id)).split() print "EventService ip: ", self.event_service_ip self.event_service_id = int(s) self.subscriber()
def add_node(self, url, create=False, root=None): """Adds a new node, with the given url/name""" if create: dir = os.path.join(self.basedir, url) app = sync.Application(dir=dir) self.subnodes[url] = app node = SubNode(url) node.added(self.ring.nodes, backups=self.backups, root=root) self.ring = HashRing(self.ring.nodes + [url])
def configure(self, options, config): self.node_count = options.distributed_nodes self.node_id = options.distributed_node_number if not self._options_are_valid(): self.enabled = False return if options.distributed_disabled: self.enabled = False return if self.node_count > 1: # If the user gives us a non-1 count of distributed nodes, then # let's distribute their tests self.enabled = True self.hash_ring = HashRing(range(1, self.node_count + 1))
class myHashRing(): def __init__(self, add): self.ring = HashRing(add) def getNode(self, team): return self.ring.get_node(team) def reHash(self, add): self.ring = HashRing(add)
def __init__(self, port): self.connList = [] self.port = port # socket vai usar tcp self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # reusar o mesmo socket para outras conexões self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # classe que vai fazer consistent hashing dos supernós self.ring = HashRing()
def __init__(self): self._ring = HashRing(DB_SERVERS) self._cat_coll = {} cnt = 0 for i in CATEGORIES: self._cat_coll.update( {CATEGORIES[i]: MongoClient(DB_SERVERS[cnt], MONGO_PORT).test}) cnt += 1 self._coll = {} for i in DB_SERVERS: name = self._get_table(i, TABLE_AUTHOR) self._coll.update( {name: MongoClient(i, MONGO_PORT).test[TABLE_AUTHOR]}) name = self._get_table(i, TABLE_CATEGORY) self._coll.update( {name: MongoClient(i, MONGO_PORT).test[TABLE_CATEGORY]}) if DEBUG: self._upload_cnt = 0
class ShardClient(object): """ ::code-block servers = [ {'name': 'server1', 'host': '192.168.0.246', 'port': 6379, 'db': 0, 'weight': 1}, {'name': 'server2', 'host': '192.168.0.247', 'port': 6379, 'db': 0, 'weight': 1}, {'name': 'server3', 'host': '192.168.0.248', 'port': 6379, 'db': 0, 'weight': 1}, {'name': 'server4', 'host': '192.168.0.249', 'port': 6379, 'db': 0, 'weight': 1}, ] """ def __init__(self, servers): self._connections = {} hosts, weights = self.format_servers(servers) self._ring = HashRing(hosts, weights) self.build_connections(servers) def format_servers(self, servers): hosts = [] weights = {} for s in servers: name = self.node_name(s) hosts.append(name) weights[name] = s['weight'] return hosts, weights def node_name(self, s): return ('%s:%s:%s:%s')%(s['name'], s['host'], s['port'], s['db']) def build_connections(self, servers): for s in servers: self._connections[self.node_name(s)] = self.connect_redis(**s) def connect_redis(self, host='localhost', port=6379, db=0, **kwargs): return redis.StrictRedis(host=host, port=port, db=db) def get_server(self, key): return self._ring.get_node(key) def get_connection(self, key): node = self._ring.get_node(key) return self._connections[node]
def remove_self(self, req): """Responds to ``POST /remove-self`` This is a request for this node to gracefully remove itself. It will attempt to back up its data to the other nodes that should take over. This takes a request with the JSON data: `name`: the name of this node `other`: a list of all nodes (including this) `backups`: the number of backups to make It responds with a text description of what it did. """ self.assert_is_internal(req) status = Response(content_type='text/plain') self.storage.disable() data = req.json self_name = data['name'] status.write('Disabling node %s\n' % self_name) ring = HashRing(data['other']) for domain, username, bucket in self.storage.all_dbs(): assert bucket.startswith('/') path = '/' + domain + '/' + username + bucket db = self.storage.for_user(domain, username, bucket) if db.is_deprecated: db.clear() continue iterator = iter(ring.iterate_nodes(path)) active_nodes = [iterator.next() for i in xrange(data['backups'] + 1)] new_node = iterator.next() assert self_name in active_nodes, '%r not in %r' % (self_name, active_nodes) status.write('Sending %s to node %s\n' % (path, new_node)) url = urlparse.urljoin(req.application_url, '/' + new_node) send = Request.blank(url + urllib.quote(path) + '?paste', method='POST', body=''.join(db.encode_db())) send.environ['cutout.root'] = req.environ.get('cutout.root') resp = forward(send) assert resp.status_code == 201, str(resp) status.write(' success, deleting\n') db.clear() self.storage.clear() return status
class DistRedis(MethodMissing): def __init__(self,hosts): self.hosts=[] for h in hosts: host,port = h.split(':') self.hosts.append(Redis(host,int(port))) self.ring = HashRing(self.hosts) def add_server(self,server): server,port = server.split(':') r= Redis(server,port) self.ring.add_node(r) def save(self): for redis in self.ring: redis.save() def bgsave(self): for redis in self.ring: redis.save(True) def delete_cloud(self): for redis in self.ring: for key in self.ring.keys("*"): redis.delete(key) def quit(self): for redis in self.ring: redis.quit def node_for_key(self,key): if re.match("/\{(.*)\?\}/",key): l=re.split("/\{(.*)\?\}/",key) key = l[0] return self.ring.get_node(key) def method_missing(self, attr, *args, **kwargs): redis = self.node_for_key(args[0]) if redis != None: return redis.__getattribute__(attr)(*args,**kwargs) '''def node_for_key(self,key):
def remove_node(self, url, root=None, force=False): """Removes the given node (according to its url/name) If force=True then the node is removed without its cooperation """ node = SubNode(url) node.remove(self.ring.nodes, backups=self.backups, force=force, root=root) new_nodes = list(self.ring.nodes) new_nodes.remove(url) self.ring = HashRing(new_nodes)
class ShardClient(object): """ ::code-block servers = [ {'name': 'server1', 'host': '192.168.0.246', 'port': 6379, 'db': 0, 'weight': 1}, {'name': 'server2', 'host': '192.168.0.247', 'port': 6379, 'db': 0, 'weight': 1}, {'name': 'server3', 'host': '192.168.0.248', 'port': 6379, 'db': 0, 'weight': 1}, {'name': 'server4', 'host': '192.168.0.249', 'port': 6379, 'db': 0, 'weight': 1}, ] """ def __init__(self, servers): self._connections = {} hosts, weights = self.format_servers(servers) self._ring = HashRing(hosts, weights) self.build_connections(servers) def format_servers(self, servers): hosts = [] weights = {} for s in servers: name = self.node_name(s) hosts.append(name) weights[name] = s['weight'] return hosts, weights def node_name(self, s): return ('%s:%s:%s:%s') % (s['name'], s['host'], s['port'], s['db']) def build_connections(self, servers): for s in servers: self._connections[self.node_name(s)] = self.connect_redis(**s) def connect_redis(self, host='localhost', port=6379, db=0, **kwargs): return redis.StrictRedis(host=host, port=port, db=db) def get_server(self, key): return self._ring.get_node(key) def get_connection(self, key): node = self._ring.get_node(key) return self._connections[node]
def __init__(self, peer=None, port=9900): self.address = socket.gethostbyname(socket.gethostname()) self.port = port self.peer = peer self.ring = HashRing() try: ping(self.location) print 'Uh-oh. Our location responded to a ping!' raise socket.error(43, 'Address already in use') except NodeNotFound: pass
class MultiserverClient(Client): """Client that connects to multiple servers""" def __init__(self, servers=servers): self.servers = servers self.ring = HashRing(servers) #################### PRIVATE FUNCTIONS #################### def base_url(self, key): server = self.ring.get_node(key) # print "Using server ", server, " for key ", key return 'http://%s' % (server)
def split_keys(self,keylocation): keys_1= {} keys_2= {} api=MugenDBAPI('temp_dbfile','temp_keyfile') for_new = MugenDBAPI('MugenDBfile_{}.txt'.format(self.newnode[0]),'KeyMap_{}.txt'.format(self.newnode[0])) servers = [self.slavenum+":10000",self.newnode[1]+":10000"] ring = HashRing(servers) for key in keylocation: with open("MugenDBfile.txt",'r') as myfile: myfile.seek(keylocation[key][1],0) data = json.loads(myfile.readline()) hxmd5=calculatemd5(key) server=ring.get_node(hxmd5) if server == servers[1]: for_new.put(key,data,keys_2,keylocation[key][0]) else: api.put(key,data,keys_1,keylocation[key][0]) os.system('mv temp_dbfile.txt MugenDBfile.txt ') os.system('mv temp_keyfile.txt KeyMap.txt ')
def __init__(self, hosts, marshal_module, key_prefix='', default_timeout=300): """port, db, password, weight are optional marshal_module should contain standard python marshal method :loads: :dumps: hosts examples: [ 'redis://localhost', 'redis://*****:*****@localhost:6382/2', ] or with weights (all hosts' weights must be provided together!) [ ('redis://localhost', 2), ('redis://localhost:6380/', 5), # ... ] """ self.key_prefix = key_prefix self.default_timeout = default_timeout self.marshal_module = marshal_module if len(hosts[0]) > 1: nodes = [x for x in map(lambda _: _[0], hosts)] weights = {x: y for x, y in hosts} else: nodes = hosts weights = None self.ring = HashRing(nodes, weights) self.clients = {} self._nodes = nodes
def ring_slow(): r = HashRing() nodes = [] for i in range(num_nodes): r.add_node(b'test-%i' % i, num_replicas=512) r.add_nodes(nodes)
def main(): servers = [ '192.168.0.235', '192.168.0.237', '192.168.0.239', '192.168.0.241', '192.168.0.243', '192.168.0.245', '192.168.0.247', '192.168.0.249' ] keys = [ "my_key", "FooBarBaz", "my_key", "FooBarBaz", "_01234567890-", "_01234567890-", "@#$%^&*!()[]{}", "timekeeper", "highball", "ponytail", "candlestick", "watermelon", "timepieces", "forewarn", "newborn", "firebreak", "disk drive", "warehouse", "forget", "bypass", "crossbow", "supernatural", "paul-frederic-simon", "deprive", "gregarious", "procrastination", "suffice" ] weights = { '192.168.0.235': 1, '192.168.0.237': 2, '192.168.0.239': 1, '192.168.0.241': 1, '192.168.0.243': 1, '192.168.0.245': 2, '192.168.0.247': 1, '192.168.0.249': 3 } ring = HashRing(servers) for key in keys: server = ring.get_node(key) print(server) ring = HashRing(servers, weights) for key in keys: server = ring.get_node(key) print(server)
class ConsistentHashing(object): ''' Usage: serverWithWeights = { '192.168.0.1:11212': 1, '192.168.0.2:11212': 3, '192.168.0.3:11212': 2, } ring = ConsistentHashing(serverWithWeights) server = ring.getNode("key") ''' def __init__(self, nodeWithWeights): super(ConsistentHashing, self).__init__() self.__ring = HashRing(nodeWithWeights.keys(), nodeWithWeights) def getNode(self, key): return self.__ring.get_node(key) def getNodeIterator(self, key, distinct = True): return self.__ring.iterate_nodes(key, distinct)
def __init__(self, preload=None, preload_dir=None, backups=1): self.subnodes = {} self.basedir = preload_dir nodes = [] if preload: for i in xrange(preload): name = 'node-%03i' % i dir = os.path.join(preload_dir, name) app = sync.Application(dir=dir) self.subnodes[name] = app nodes.append(name) self.ring = HashRing(nodes) self.backups = backups
def test_hash_ring_basic_flow(): ring = HashRing() # A ring with no node can't find any nodes. assert ring.find_node(1) is None # Adding a node should find the node. assert ring.add_node(b'test') assert ring.find_node(1) == b'test' # Removing it should work. assert ring.remove_node(b'test') assert ring.find_node(1) is None
def test_ring_find_nodes(benchmark, num_nodes): r = HashRing() nodes = [] for i in range(num_nodes): r.add_node(b'test-%i' % i, num_replicas=512) r.add_nodes(nodes) def ring_lookup(): return r.find_nodes(b'hello', 3) assert benchmark(ring_lookup) == r.find_nodes(b'hello', 3)
def __init__(self,): ''' init redis with the config. start a inter thread. hashring with replication inworker is a thread can push or get worker from it. ''' self.rs=redis.StrictRedis(host=config.redis_host, port=config.redis_port,db=0) self.inter=inputer.Inputer(config.input_home) self.db=Filedb(config.db_file) self.workers=self.db.get() self.time=time.time() #TODO:add the weight for hashring() init self.hashring=HashRing(nodes=self.workers,replicas=100) self.nodes={}
def generate_ring(nodes, name_to_obj, node_type=None): """Given a set of nodes it created nodes's and returns a hash ring with them""" global default_node if not node_type: node_type = default_node objects = [] for name in nodes: obj = node_type(name, nodes[name]) name_to_obj[name] = obj objects.append(obj) return HashRing(objects)