def __init__(self, join_ip, topic="topic"):
self.listID = []
for i in range(0, 256):
self.listID.append(i)
ring = HashRing(self.listID)
self.id = ring.get_node(topic)
print "id assigned:", self.id
self.join_ip = join_ip
self.topic = topic
self.context = zmq.Context()
s, self.event_service_ip = self.call_remote_procedure(
self.join_ip, "findSuccessor", str(self.id)).split()
print "EventService ip: ", self.event_service_ip
self.event_service_id = int(s)
xsub_url = "tcp://" + self.event_service_ip + ":5556"
socket = self.context.socket(zmq.PUB)
socket.connect(xsub_url)
# pub.bind(xpub_url)
for n in range(1000):
priority = randint(0, 9)
msg = [topic, str(priority), str(n)]
socket.send_multipart(msg)
print "for the message %s with priority %d" % (
topic, priority), "process time in publisher is", time.clock()
time.sleep(1.25)
def __init__(self, addr, port):
RPCServer.__init__(self, addr, port)
len_up = len(UPLOAD_SERVERS)
len_repo = len(REPOSITORY_SERVERS)
if len_up < len_repo or len_up % len_repo != 0:
show_error(self, 'failed to initialize')
raise Exception('failed to initialize')
addr = localhost()
if addr not in REPOSITORY_SERVERS:
show_error(self, 'failed to initialize')
raise Exception('failed to initialize REPOSITORY_SERVERS')
for i in range(len(REPOSITORY_SERVERS)):
if addr == REPOSITORY_SERVERS[i]:
break
total = len_up / len_repo
self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total]
self._print('upload_servers=%s' % str(self._upload_servers))
if HDFS:
self._port = HDFS_PORT
self._client = HDFSClient()
else:
self._port = FTP_PORT
self._client = FTPClient()
self._server = FTPServer()
if REPO_DB:
self._db = Database(addr=REPO_DB)
else:
self._db = Database(addr=addr)
locks = []
for _ in range(LOCK_MAX):
locks.append(Lock())
self._locks = HashRing(locks)
if DEBUG:
self._upload_cnt = 0
self._download_cnt = 0
def ring_fast():
r = HashRing()
nodes = []
for i in range(num_nodes):
nodes.append(HashRingNode(b'test-%i' % i, num_replicas=512))
r.add_nodes(nodes)
def __init__(self, hosts, connect=True):
"""
@param hosts: A C{list} of C{tuple}s containing hosts and ports.
"""
self.hosts = hosts
self.hash_ring = HashRing(hosts)
if connect:
self.connect()
def getClient(self, key):
hosts = self.getActiveConnections()
if hosts != self.hash_ring.nodes:
self.hash_ring = HashRing(hosts)
log.msg("Using %i active hosts" % len(hosts))
if len(hosts) == 0:
raise NoServerError("No connected servers remaining.")
return self.hash_ring.get_node(key)
def __init__(self, servers, *k, **kw):
self.hash_ring = HashRing(servers)
memcache.Client.__init__(self, servers, *k, **kw)
self.server_mapping = {}
for server_uri, server_obj in zip(servers, self.servers):
self.server_mapping[server_uri] = server_obj
def __init__(self, nodes):
"""构造函数
nodes 节点数据，格式：{'node01': resource01, 'node02': resource02}
"""
self._nodes = nodes
ids = nodes.keys()
ids.sort()
self._ring = HashRing(ids)
def __init__(self, location=None, base_url=None, **kwargs):
self.clients = {}
for server in settings.DISTRIBUTED_MOGILEFS_CONFIG['SERVERS']:
srv = settings.DISTRIBUTED_MOGILEFS_CONFIG['SERVERS'][server]
self.clients[server] = Client(domain=srv['DOMAIN'],
trackers=srv['TRACKERS'])
self.servers = settings.DISTRIBUTED_MOGILEFS_CONFIG['SERVERS']
self.ring = HashRing(settings.DISTRIBUTED_MOGILEFS_CONFIG['SLOTS'])
self.kwargs = kwargs
def __init__(self, hosts, timeout=5, lazy=False):
self._hosts = hosts
self._timeout = timeout
self._sockets = None
self._ring = HashRing(self._hosts)
if not lazy:
self.open()
def create_sets(servers):
server_sets = {}
for s in servers:
server_sets[s] = set()
ring = HashRing(servers)
for word in palindromes:
node = ring.get_node(word)
server_sets[node].add(word)
return server_sets
def create_sets(servers):
server_sets = {}
for s in servers:
server_sets[s] = set()
ring = HashRing(servers)
for word in palindromes:
node = ring.get_node(word)
server_sets[node].add(word)
return server_sets
def test_iterate_nodes():
simple_list = ['1', '2', '3', '4', '5']
new_ring = HashRing(simple_list)
nodes = []
for node in new_ring.iterate_nodes('a'):
nodes.append(node)
assert len(nodes) == len(simple_list)
for elm in simple_list:
assert elm in nodes
def test_iterate_nodes():
simple_list = ['1', '2', '3', '4', '5']
new_ring = HashRing(simple_list)
nodes = []
for node in new_ring.iterate_nodes('a'):
nodes.append(node)
assert len(nodes) == len(simple_list)
for elm in simple_list:
assert elm in nodes
def take_over(self, req):
"""Attached to ``POST /take-over``
Takes over databases from another server, that presumably has
gone offline without notice.
This goes through all of the local databases, and sees if this
node was either using the bad node as a backup, or is a backup
for the bad node. In either case it finds the new node that
should be either master or handling the bad node, and sends
the local database to that server.
Takes a JSON body with keys:
`other`: a list of all nodes
`name`: the name of *this* node
`bad`: the bad node being removed
`backups`: the number of backups
"""
self.assert_is_internal(req)
status = Response(content_type='text/plain')
data = req.json
nodes = data['other']
self_name = data['name']
bad_node = data['bad']
assert self_name != bad_node
backups = data['backups']
ring = HashRing(nodes)
for domain, username, bucket in self.storage.all_dbs():
assert bucket.startswith('/')
path = '/' + domain + '/' + username + bucket
iterator = iter(ring.iterate_nodes(path))
active_nodes = [iterator.next() for i in xrange(backups + 1)]
replacement_node = iterator.next()
# Not all the backups should try to restore the database, so instead
# just the "first" does it
restore = False
if active_nodes[0] == bad_node and active_nodes[1:] and active_nodes[1] == self_name:
status.write('Master node %s for %s removed\n' % (bad_node, path))
restore = True
elif bad_node in active_nodes and active_nodes[0] == self_name:
status.write('Backup node %s for %s removed\n' % (bad_node, path))
restore = True
if not restore:
continue
db = self.storage.for_user(domain, username, bucket)
send = Request.blank(replacement_node + urllib.quote(path) + '?paste',
method='POST', body=''.join(db.encode_db()))
send.environ['cutout.root'] = req.environ.get('cutout.root')
resp = forward(send)
assert resp.status_code == 201, str(resp)
#status.write(' nodes: %r - %r / %r\n' % (active_nodes, bad_node, self_name))
status.write(' success, added to %s (from %s)\n' % (replacement_node, self_name))
return status
class LocatorHandler(BaseHandler, Locator.Iface):
def __init__(self, peer=None, port=9900):
self.address = socket.gethostbyname(socket.gethostname())
self.port = port
self.peer = peer
self.ring = HashRing()
try:
ping(self.location)
print 'Uh-oh. Our location responded to a ping!'
raise socket.error(43, 'Address already in use')
except NodeNotFound:
pass
@property
def here(self):
"Give the canonical string representation"
return loc2str(self)
@property
def location(self):
"Give the canonical Location"
return Location(address=self.address, port=self.port)
def join(self, location):
"""
Parameters:
- location
"""
self.add(location, [self.location])
ping_until_return(location)
items = self.ring.nodes.difference([loc2str(location)])
def remove(self, location, authorities):
"""
Parameters:
- location
- authorities
"""
key = loc2str(location)
self.ring.remove(loc2str(location))
authorities.append(self.location)
destinations = select_peers(self.ring.nodes.difference(map(loc2str,authorities)))
for destination in destinations:
try:
remote_call('remove', str2loc(destination), location, authorities)
break
except NodeNotFound, tx:
# enter all nodes as authorities to avoid race conditions
# lazy invalidation
self.remove(tx.location, map(str2loc, self.ring.nodes))
print "removed %s:%d" % (location.address, location.port)
def url(self, name, node=None, **kwargs):
name = self.path(name)
if not name:
return ""
node_key = node or self.get_key_by_name(name, **kwargs)
name = name.lstrip("/")
urls = self.servers[node_key]["URLS"]
urls = [urls] if isinstance(urls, basestring) else urls
if len(urls) > 1:
ring2 = HashRing(urls)
base_url = ring2.get_node(node_key)
else:
base_url = urls[0]
return base_url + name
def __init__(self, addr, port):
RPCServer.__init__(self, addr, port, user=User())
self._sandbox = Sandbox()
locks = []
for _ in range(LOCK_MAX):
locks.append(Lock())
self._locks = HashRing(locks)
self._cache = {}
if DEBUG:
self._register_cnt = 0
self._login_cnt = 0
self._upload_cnt = 0
self._install_cnt = 0
self._uninstall_cnt = 0
def url(self, name, node=None, **kwargs):
name = self.path(name)
if not name:
return ''
node_key = node or self.get_key_by_name(name, **kwargs)
name = name.lstrip('/')
urls = self.servers[node_key]['URLS']
urls = [urls] if isinstance(urls, basestring) else urls
if len(urls) > 1:
ring2 = HashRing(urls)
base_url = ring2.get_node(node_key)
else:
base_url = urls[0]
return base_url + name
def test_with_objects():
simple_list = [Server(1), Server(2), Server(3)]
new_ring = HashRing(simple_list)
node = new_ring.get_node('BABU')
assert node in simple_list
nodes = []
for node in new_ring.iterate_nodes('aloha'):
nodes.append(node)
assert len(nodes) == len(simple_list)
for elm in simple_list:
assert elm in nodes
def test_with_objects():
simple_list = [Server(1), Server(2), Server(3)]
new_ring = HashRing(simple_list)
node = new_ring.get_node('BABU')
assert node in simple_list
nodes = []
for node in new_ring.iterate_nodes('aloha'):
nodes.append(node)
assert len(nodes) == len(simple_list)
for elm in simple_list:
assert elm in nodes
class MemcacheRing(memcache.Client):
"""Extends python-memcache so it uses consistent hashing to
distribute the keys.
"""
def __init__(self, servers, *k, **kw):
self.hash_ring = HashRing(servers)
memcache.Client.__init__(self, servers, *k, **kw)
self.server_mapping = {}
for server_uri, server_obj in zip(servers, self.servers):
self.server_mapping[server_uri] = server_obj
def _get_server(self, key):
if isinstance(key, tuple):
return memcache.Client._get_server(key)
for i in range(self._SERVER_RETRIES):
iterator = self.hash_ring.iterate_nodes(key)
for server_uri in iterator:
server_obj = self.server_mapping[server_uri]
if server_obj.connect():
return server_obj, key
return None, None
class MemcacheRing(memcache.Client):
"""Extends python-memcache so it uses consistent hashing to
distribute the keys.
"""
def __init__(self, servers, *k, **kw):
self.hash_ring = HashRing(servers)
memcache.Client.__init__(self, servers, *k, **kw)
self.server_mapping = {}
for server_uri, server_obj in zip(servers, self.servers):
self.server_mapping[server_uri] = server_obj
def _get_server(self, key):
if type(key) == types.TupleType:
return memcache.Client._get_server(key)
for i in range(self._SERVER_RETRIES):
iterator = self.hash_ring.iterate_nodes(key)
for server_uri in iterator:
server_obj = self.server_mapping[server_uri]
if server_obj.connect():
return server_obj, key
return None, None
def __init__(self, query):
Thread.__init__(self)
self._pool = ThreadPool(processes=4)
self._workers = HashRing(WORKER_SERVERS)
self._identity = bytes(uuid.uuid4())
self._query = query
self._init_sock()
def receive_slave_failure(self): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) host = socket.gethostname() print 'receiving failure' sock.bind((self.host, 10012)) global backup_slave_nodes while True: request, addr = sock.recvfrom(1024) #request will be just the slave number dead_slaves = json.loads(request) print 'Recieved dead slaves : ' print dead_slaves for slave in dead_slaves: if slave in self.slave_nodes.keys(): endpoint = self.slave_nodes[slave] del self.slave_nodes[slave] del self.backup[slave] self.memcache_servers.remove(endpoint) print 'after deletion'+str(self.slave_nodes) #Now add the backup slave nodes self.slave_nodes[slave]=backup_slave_nodes[slave] self.backup[slave]=backup_slave_nodes[slave] self.memcache_servers.append(backup_slave_nodes[slave]) print 'after addition'+str(self.slave_nodes) #change the ring using the new slave self.ring = HashRing(self.memcache_servers)
def __init__(self, conns, pipelines=None):
length = len(conns)
if length not in ShardRedis.cache:
ShardRedis.cache[length] = HashRing(range(length))
self.ring = ShardRedis.cache[length]
self.conns = conns
self.pipelines = pipelines
def __init__(self, addr, port):
RPCServer.__init__(self, addr, port)
len_up = len(UPLOAD_SERVERS)
len_repo = len(REPOSITORY_SERVERS)
if len_up < len_repo or len_up % len_repo != 0:
show_error(self, 'failed to initialize')
raise Exception('failed to initialize')
addr = localhost()
if addr not in REPOSITORY_SERVERS:
show_error(self, 'failed to initialize')
raise Exception('failed to initialize REPOSITORY_SERVERS')
for i in range(len(REPOSITORY_SERVERS)):
if addr == REPOSITORY_SERVERS[i]:
break
total = len_up / len_repo
self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total]
self._print('upload_servers=%s' % str(self._upload_servers))
if HDFS:
self._port = HDFS_PORT
self._client = HDFSClient()
else:
self._port = FTP_PORT
self._client = FTPClient()
self._server = FTPServer()
if REPO_DB:
self._db = Database(addr=REPO_DB)
else:
self._db = Database(addr=addr)
locks = []
for _ in range(LOCK_MAX):
locks.append(Lock())
self._locks = HashRing(locks)
if DEBUG:
self._upload_cnt = 0
self._download_cnt = 0
def __init__(self,portNumber):
self.portNumber = portNumber
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.host = socket.gethostname()
print self.host
self.sock.bind((self.host, self.portNumber))
self.monitors = dict()
with open("config/monitor.txt") as myfile:
for line in myfile:
name, endpoint = line.partition("=")[::2]
print endpoint
self.monitors[name] = endpoint
self.pool = ThreadPool(10) #TODO : configure this
self.memcache_servers = []
self.slave_nodes = dict()
self.backup = dict()
self.config = {"result":"New","host":self.host,"port":self.portNumber}
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
host,port = self.monitors[self.monitors.keys()[0]].split(":")
sock.sendto(json.dumps(self.config), (host,int(port)))
slave_config,addr = self.sock.recvfrom(1024)
sock.close()
with open("config/slave.txt",'w') as fin:
for val in slave_config:
fin.write(str(val))
with open("config/slave.txt",'r') as myfile:
for line in myfile:
name, endpoints = line.partition("=")[::2]
endpoint1,endpoint2= endpoints.split(',')
self.memcache_servers.append(endpoint1)
self.slave_nodes[name]=endpoint1
self.backup[name] =endpoint2
self.ring = HashRing(self.memcache_servers)
print self.backup
def query_deprecate(self, req):
"""Responds to ``POST /query-deprecate``
This is used when a new node is added to the system, and all
existing nodes are asked for what databases should be assigned
to the new node. Any such database will be deprecated, and a
list of those databases is returned.
Accepts a JSON body with the keys:
`other`: list of all nodes
`name`: the name of this node
`new`: the node being added
`backups`: the number of backups to keep
Returns JSON::
{"deprecated": [deprecated items]}
Where the deprecated items are::
{"path": "/domain/user/bucket",
"domain": "domain",
"username": "******",
"bucket": "bucket"
}
"""
self.assert_is_internal(req)
data = req.json
nodes = data['other']
self_name = data['name']
new_node = data['new']
backups = data['backups']
ring = HashRing(nodes + [new_node])
deprecated = []
for domain, username, bucket in self.storage.all_dbs():
assert bucket.startswith('/')
path = '/' + domain + '/' + username + bucket
iterator = iter(ring.iterate_nodes(path))
active_nodes = [iterator.next() for i in xrange(backups + 1)]
deprecated_node = iterator.next()
if deprecated_node == self_name and new_node in active_nodes:
deprecated.append(
{'path': path, 'domain': domain, 'username': username, 'bucket': bucket})
db = self.storage.for_user(domain, username, bucket)
db.deprecate()
return Response(json={'deprecated': deprecated})
def __init__(self, servers, *k, **kw):
self.hash_ring = HashRing(servers)
memcache.Client.__init__(self, servers, *k, **kw)
self.server_mapping = {}
for server_uri, server_obj in zip(servers, self.servers):
self.server_mapping[server_uri] = server_obj
def getClient(self, key):
hosts = self.getActiveConnections()
if hosts != self.hash_ring.nodes:
self.hash_ring = HashRing(hosts)
log.msg("Using %i active hosts" % len(hosts))
if len(hosts) == 0:
raise NoServerError("No connected servers remaining.")
return self.hash_ring.get_node(key)
def __init__(self, location=None, base_url=None, **kwargs):
self.clients = {}
for server in settings.DISTRIBUTED_MOGILEFS_CONFIG["SERVERS"]:
srv = settings.DISTRIBUTED_MOGILEFS_CONFIG["SERVERS"][server]
self.clients[server] = Client(domain=srv["DOMAIN"], trackers=srv["TRACKERS"])
self.servers = settings.DISTRIBUTED_MOGILEFS_CONFIG["SERVERS"]
self.ring = HashRing(settings.DISTRIBUTED_MOGILEFS_CONFIG["SLOTS"])
self.kwargs = kwargs
def __init__(self, hosts, connect=True):
"""
@param hosts: A C{list} of C{tuple}s containing hosts and ports.
"""
self.hosts = hosts
self.hash_ring = HashRing(hosts)
if connect:
self.connect()
def __init__(self, join_ip, topic="topic"):
self.listID = []
for i in range(0, 256):
self.listID.append(i)
ring = HashRing(self.listID)
self.id = ring.get_node(topic)
print "id assigned:", self.id
self.join_ip = join_ip
self.topic = topic
self.context = zmq.Context()
s, self.event_service_ip = self.call_remote_procedure(
self.join_ip, "findSuccessor", str(self.id)).split()
print "EventService ip: ", self.event_service_ip
self.event_service_id = int(s)
self.subscriber()
def __init__(self, hosts, timeout=5, lazy=False):
self._hosts = hosts
self._timeout = timeout
self._sockets = None
self._ring = HashRing(self._hosts)
if not lazy:
self.open()
def add_node(self, url, create=False, root=None):
"""Adds a new node, with the given url/name"""
if create:
dir = os.path.join(self.basedir, url)
app = sync.Application(dir=dir)
self.subnodes[url] = app
node = SubNode(url)
node.added(self.ring.nodes, backups=self.backups, root=root)
self.ring = HashRing(self.ring.nodes + [url])
def configure(self, options, config):
self.node_count = options.distributed_nodes
self.node_id = options.distributed_node_number
if not self._options_are_valid():
self.enabled = False
return
if options.distributed_disabled:
self.enabled = False
return
if self.node_count > 1:
# If the user gives us a non-1 count of distributed nodes, then
# let's distribute their tests
self.enabled = True
self.hash_ring = HashRing(range(1, self.node_count + 1))
class myHashRing():
def __init__(self, add):
self.ring = HashRing(add)
def getNode(self, team):
return self.ring.get_node(team)
def reHash(self, add):
self.ring = HashRing(add)
def __init__(self, port):
self.connList = []
self.port = port
# socket vai usar tcp
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# reusar o mesmo socket para outras conexões
self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# classe que vai fazer consistent hashing dos supernós
self.ring = HashRing()
def __init__(self):
self._ring = HashRing(DB_SERVERS)
self._cat_coll = {}
cnt = 0
for i in CATEGORIES:
self._cat_coll.update(
{CATEGORIES[i]: MongoClient(DB_SERVERS[cnt], MONGO_PORT).test})
cnt += 1
self._coll = {}
for i in DB_SERVERS:
name = self._get_table(i, TABLE_AUTHOR)
self._coll.update(
{name: MongoClient(i, MONGO_PORT).test[TABLE_AUTHOR]})
name = self._get_table(i, TABLE_CATEGORY)
self._coll.update(
{name: MongoClient(i, MONGO_PORT).test[TABLE_CATEGORY]})
if DEBUG:
self._upload_cnt = 0
class ShardClient(object):
"""
::code-block
servers = [
{'name': 'server1', 'host': '192.168.0.246', 'port': 6379, 'db': 0, 'weight': 1},
{'name': 'server2', 'host': '192.168.0.247', 'port': 6379, 'db': 0, 'weight': 1},
{'name': 'server3', 'host': '192.168.0.248', 'port': 6379, 'db': 0, 'weight': 1},
{'name': 'server4', 'host': '192.168.0.249', 'port': 6379, 'db': 0, 'weight': 1},
]
"""
def __init__(self, servers):
self._connections = {}
hosts, weights = self.format_servers(servers)
self._ring = HashRing(hosts, weights)
self.build_connections(servers)
def format_servers(self, servers):
hosts = []
weights = {}
for s in servers:
name = self.node_name(s)
hosts.append(name)
weights[name] = s['weight']
return hosts, weights
def node_name(self, s):
return ('%s:%s:%s:%s')%(s['name'], s['host'], s['port'], s['db'])
def build_connections(self, servers):
for s in servers:
self._connections[self.node_name(s)] = self.connect_redis(**s)
def connect_redis(self, host='localhost', port=6379, db=0, **kwargs):
return redis.StrictRedis(host=host, port=port, db=db)
def get_server(self, key):
return self._ring.get_node(key)
def get_connection(self, key):
node = self._ring.get_node(key)
return self._connections[node]
def remove_self(self, req):
"""Responds to ``POST /remove-self``
This is a request for this node to gracefully remove itself.
It will attempt to back up its data to the other nodes that
should take over.
This takes a request with the JSON data:
`name`: the name of this node
`other`: a list of all nodes (including this)
`backups`: the number of backups to make
It responds with a text description of what it did.
"""
self.assert_is_internal(req)
status = Response(content_type='text/plain')
self.storage.disable()
data = req.json
self_name = data['name']
status.write('Disabling node %s\n' % self_name)
ring = HashRing(data['other'])
for domain, username, bucket in self.storage.all_dbs():
assert bucket.startswith('/')
path = '/' + domain + '/' + username + bucket
db = self.storage.for_user(domain, username, bucket)
if db.is_deprecated:
db.clear()
continue
iterator = iter(ring.iterate_nodes(path))
active_nodes = [iterator.next() for i in xrange(data['backups'] + 1)]
new_node = iterator.next()
assert self_name in active_nodes, '%r not in %r' % (self_name, active_nodes)
status.write('Sending %s to node %s\n' % (path, new_node))
url = urlparse.urljoin(req.application_url, '/' + new_node)
send = Request.blank(url + urllib.quote(path) + '?paste',
method='POST', body=''.join(db.encode_db()))
send.environ['cutout.root'] = req.environ.get('cutout.root')
resp = forward(send)
assert resp.status_code == 201, str(resp)
status.write(' success, deleting\n')
db.clear()
self.storage.clear()
return status
class DistRedis(MethodMissing):
def __init__(self,hosts):
self.hosts=[]
for h in hosts:
host,port = h.split(':')
self.hosts.append(Redis(host,int(port)))
self.ring = HashRing(self.hosts)
def add_server(self,server):
server,port = server.split(':')
r= Redis(server,port)
self.ring.add_node(r)
def save(self):
for redis in self.ring:
redis.save()
def bgsave(self):
for redis in self.ring:
redis.save(True)
def delete_cloud(self):
for redis in self.ring:
for key in self.ring.keys("*"):
redis.delete(key)
def quit(self):
for redis in self.ring:
redis.quit
def node_for_key(self,key):
if re.match("/\{(.*)\?\}/",key):
l=re.split("/\{(.*)\?\}/",key)
key = l[0]
return self.ring.get_node(key)
def method_missing(self, attr, *args, **kwargs):
redis = self.node_for_key(args[0])
if redis != None:
return redis.__getattribute__(attr)(*args,**kwargs)
'''def node_for_key(self,key):
def remove_node(self, url, root=None, force=False):
"""Removes the given node (according to its url/name)
If force=True then the node is removed without its cooperation
"""
node = SubNode(url)
node.remove(self.ring.nodes, backups=self.backups, force=force, root=root)
new_nodes = list(self.ring.nodes)
new_nodes.remove(url)
self.ring = HashRing(new_nodes)
class ShardClient(object):
"""
::code-block
servers = [
{'name': 'server1', 'host': '192.168.0.246', 'port': 6379, 'db': 0, 'weight': 1},
{'name': 'server2', 'host': '192.168.0.247', 'port': 6379, 'db': 0, 'weight': 1},
{'name': 'server3', 'host': '192.168.0.248', 'port': 6379, 'db': 0, 'weight': 1},
{'name': 'server4', 'host': '192.168.0.249', 'port': 6379, 'db': 0, 'weight': 1},
]
"""
def __init__(self, servers):
self._connections = {}
hosts, weights = self.format_servers(servers)
self._ring = HashRing(hosts, weights)
self.build_connections(servers)
def format_servers(self, servers):
hosts = []
weights = {}
for s in servers:
name = self.node_name(s)
hosts.append(name)
weights[name] = s['weight']
return hosts, weights
def node_name(self, s):
return ('%s:%s:%s:%s') % (s['name'], s['host'], s['port'], s['db'])
def build_connections(self, servers):
for s in servers:
self._connections[self.node_name(s)] = self.connect_redis(**s)
def connect_redis(self, host='localhost', port=6379, db=0, **kwargs):
return redis.StrictRedis(host=host, port=port, db=db)
def get_server(self, key):
return self._ring.get_node(key)
def get_connection(self, key):
node = self._ring.get_node(key)
return self._connections[node]
def __init__(self, peer=None, port=9900):
self.address = socket.gethostbyname(socket.gethostname())
self.port = port
self.peer = peer
self.ring = HashRing()
try:
ping(self.location)
print 'Uh-oh. Our location responded to a ping!'
raise socket.error(43, 'Address already in use')
except NodeNotFound:
pass
class MultiserverClient(Client): """Client that connects to multiple servers""" def __init__(self, servers=servers): self.servers = servers self.ring = HashRing(servers) #################### PRIVATE FUNCTIONS #################### def base_url(self, key): server = self.ring.get_node(key) # print "Using server ", server, " for key ", key return 'http://%s' % (server)
def split_keys(self,keylocation):
keys_1= {}
keys_2= {}
api=MugenDBAPI('temp_dbfile','temp_keyfile')
for_new = MugenDBAPI('MugenDBfile_{}.txt'.format(self.newnode[0]),'KeyMap_{}.txt'.format(self.newnode[0]))
servers = [self.slavenum+":10000",self.newnode[1]+":10000"]
ring = HashRing(servers)
for key in keylocation:
with open("MugenDBfile.txt",'r') as myfile:
myfile.seek(keylocation[key][1],0)
data = json.loads(myfile.readline())
hxmd5=calculatemd5(key)
server=ring.get_node(hxmd5)
if server == servers[1]:
for_new.put(key,data,keys_2,keylocation[key][0])
else:
api.put(key,data,keys_1,keylocation[key][0])
os.system('mv temp_dbfile.txt MugenDBfile.txt ')
os.system('mv temp_keyfile.txt KeyMap.txt ')
def __init__(self, hosts, marshal_module,
key_prefix='', default_timeout=300):
"""port, db, password, weight are optional
marshal_module should contain standard python marshal method
:loads:
:dumps:
hosts examples:
[
'redis://localhost',
'redis://*****:*****@localhost:6382/2',
]
or with weights (all hosts' weights must be provided together!)
[
('redis://localhost', 2),
('redis://localhost:6380/', 5),
# ...
]
"""
self.key_prefix = key_prefix
self.default_timeout = default_timeout
self.marshal_module = marshal_module
if len(hosts[0]) > 1:
nodes = [x for x in map(lambda _: _[0], hosts)]
weights = {x: y for x, y in hosts}
else:
nodes = hosts
weights = None
self.ring = HashRing(nodes, weights)
self.clients = {}
self._nodes = nodes
def ring_slow():
r = HashRing()
nodes = []
for i in range(num_nodes):
r.add_node(b'test-%i' % i, num_replicas=512)
r.add_nodes(nodes)
def main():
servers = [
'192.168.0.235', '192.168.0.237', '192.168.0.239', '192.168.0.241',
'192.168.0.243', '192.168.0.245', '192.168.0.247', '192.168.0.249'
]
keys = [
"my_key", "FooBarBaz", "my_key", "FooBarBaz", "_01234567890-",
"_01234567890-", "@#$%^&*!()[]{}", "timekeeper", "highball",
"ponytail", "candlestick", "watermelon", "timepieces", "forewarn",
"newborn", "firebreak", "disk drive", "warehouse", "forget", "bypass",
"crossbow", "supernatural", "paul-frederic-simon", "deprive",
"gregarious", "procrastination", "suffice"
]
weights = {
'192.168.0.235': 1,
'192.168.0.237': 2,
'192.168.0.239': 1,
'192.168.0.241': 1,
'192.168.0.243': 1,
'192.168.0.245': 2,
'192.168.0.247': 1,
'192.168.0.249': 3
}
ring = HashRing(servers)
for key in keys:
server = ring.get_node(key)
print(server)
ring = HashRing(servers, weights)
for key in keys:
server = ring.get_node(key)
print(server)
class ConsistentHashing(object):
'''
Usage:
serverWithWeights =
{
'192.168.0.1:11212': 1,
'192.168.0.2:11212': 3,
'192.168.0.3:11212': 2,
}
ring = ConsistentHashing(serverWithWeights)
server = ring.getNode("key")
'''
def __init__(self, nodeWithWeights):
super(ConsistentHashing, self).__init__()
self.__ring = HashRing(nodeWithWeights.keys(), nodeWithWeights)
def getNode(self, key):
return self.__ring.get_node(key)
def getNodeIterator(self, key, distinct = True):
return self.__ring.iterate_nodes(key, distinct)
def __init__(self, preload=None, preload_dir=None, backups=1):
self.subnodes = {}
self.basedir = preload_dir
nodes = []
if preload:
for i in xrange(preload):
name = 'node-%03i' % i
dir = os.path.join(preload_dir, name)
app = sync.Application(dir=dir)
self.subnodes[name] = app
nodes.append(name)
self.ring = HashRing(nodes)
self.backups = backups
def test_hash_ring_basic_flow():
ring = HashRing()
# A ring with no node can't find any nodes.
assert ring.find_node(1) is None
# Adding a node should find the node.
assert ring.add_node(b'test')
assert ring.find_node(1) == b'test'
# Removing it should work.
assert ring.remove_node(b'test')
assert ring.find_node(1) is None
def test_ring_find_nodes(benchmark, num_nodes):
r = HashRing()
nodes = []
for i in range(num_nodes):
r.add_node(b'test-%i' % i, num_replicas=512)
r.add_nodes(nodes)
def ring_lookup():
return r.find_nodes(b'hello', 3)
assert benchmark(ring_lookup) == r.find_nodes(b'hello', 3)
def __init__(self, addr, port):
RPCServer.__init__(self, addr, port, user=User())
self._sandbox = Sandbox()
locks = []
for _ in range(LOCK_MAX):
locks.append(Lock())
self._locks = HashRing(locks)
self._cache = {}
if DEBUG:
self._register_cnt = 0
self._login_cnt = 0
self._upload_cnt = 0
self._install_cnt = 0
self._uninstall_cnt = 0
def __init__(self,):
'''
init redis with the config. start a inter thread.
hashring with replication
inworker is a thread can push or get worker from it.
'''
self.rs=redis.StrictRedis(host=config.redis_host, port=config.redis_port,db=0)
self.inter=inputer.Inputer(config.input_home)
self.db=Filedb(config.db_file)
self.workers=self.db.get()
self.time=time.time()
#TODO:add the weight for hashring() init
self.hashring=HashRing(nodes=self.workers,replicas=100)
self.nodes={}
def generate_ring(nodes, name_to_obj, node_type=None):
"""Given a set of nodes it created nodes's
and returns a hash ring with them"""
global default_node
if not node_type:
node_type = default_node
objects = []
for name in nodes:
obj = node_type(name, nodes[name])
name_to_obj[name] = obj
objects.append(obj)
return HashRing(objects)
