def add_node(self, node, backup_store=None):
self.nodes.append(node)
self.hash_ring = HashRing(self.nodes)
if backup_store is not None:
self.backup_stores[node] = backup_store
else:
backup_stores_path = getattr(self, 'backup_stores_path')
if backup_stores_path is not None:
path = os.path.join(backup_stores_path, node.replace(':', '_'))
if not os.path.exists(path):
os.makedirs(path)
self.backup_stores[node] = Node(path, size=512 * 1024)
def remove_node(self, node):
self.nodes.remove(node)
self.hash_ring = HashRing(self.nodes)
backup_store = self.backup_stores[node]
obj = backup_store.get()
while obj is not None:
self.local_store.put(obj)
obj = backup_store.get()
backup_store.shutdown()
del self.backup_stores[node]
def __init__(self,
nodes,
local_node=None,
rpc_server=None,
local_store=None,
backup_stores=None,
copies=1):
self.nodes = nodes
self.local_node = local_node
self.local_store = local_store
self.rpc_server = rpc_server
self.backup_stores = backup_stores
self.hash_ring = HashRing(self.nodes)
self.copies = max(min(len(self.nodes) - 1, copies), 0)
if rpc_server is not None and \
self.local_store is not None and \
self.backup_stores is not None:
self._register_rpc()
def __init__(self, nodes, local_node=None, rpc_server=None, local_store=None, backup_stores=None, copies=1):
self.nodes = nodes
self.local_node = local_node
self.local_store = local_store
self.rpc_server = rpc_server
self.backup_stores = backup_stores
self.hash_ring = HashRing(self.nodes)
self.copies = max(min(len(self.nodes) - 1, copies), 0)
if rpc_server is not None and self.local_store is not None and self.backup_stores is not None:
self._register_rpc()
def add_node(self, node, backup_store=None):
self.nodes.append(node)
self.hash_ring = HashRing(self.nodes)
if backup_store is not None:
self.backup_stores[node] = backup_store
else:
backup_stores_path = getattr(self, "backup_stores_path")
if backup_stores_path is not None:
path = os.path.join(backup_stores_path, node.replace(":", "_"))
if not os.path.exists(path):
os.makedirs(path)
self.backup_stores[node] = Node(path, size=512 * 1024)
def remove_node(self, node):
self.nodes.remove(node)
self.hash_ring = HashRing(self.nodes)
backup_store = self.backup_stores[node]
obj = backup_store.get()
while obj is not None:
self.local_store.put(obj)
obj = backup_store.get()
backup_store.shutdown()
del self.backup_stores[node]
class Distributor(object):
"""
Given several objects, to decides which message queue node each one belong to.
"""
def __init__(self, addrs, copies=1):
self.nodes = list(addrs)
self.hash_ring = HashRing(self.nodes)
self.copies = min(copies, len(self.nodes)-1)
def distribute(self, objs):
"""
:param objs: the objects
:return: a tuple with two dicts,
the first is the mapping from the node address to the belonged objects,
the second is the mapping from the backup node address and the objects
"""
node_objs = defaultdict(list)
backup_node_objs = defaultdict(lambda: defaultdict(list))
if isinstance(objs, basestring) or not iterable(objs):
objs = [objs, ]
for obj in objs:
str_ = labelize(obj)
it = self.hash_ring.iterate_nodes(str_)
# put obj into an mq node.
put_node = next(it)
node_objs[put_node].append(obj)
for _ in xrange(self.copies):
backup_node = next(it)
if backup_node is None: continue
backup_node_objs[backup_node][put_node].append(obj)
return node_objs, backup_node_objs
def remove_node(self, addr):
if addr in self.nodes:
self.nodes.remove(addr)
self.hash_ring = HashRing(self.nodes)
def add_node(self, addr):
if addr not in self.nodes:
self.nodes.append(addr)
self.hash_ring = HashRing(self.nodes)
class Distributor(object):
def __init__(self, addrs, copies=1):
self.nodes = list(addrs)
self.hash_ring = HashRing(self.nodes)
self.copies = min(copies, len(self.nodes) - 1)
def distribute(self, objs):
node_objs = defaultdict(list)
backup_node_objs = defaultdict(lambda: defaultdict(list))
if isinstance(objs, basestring) or not iterable(objs):
objs = [
objs,
]
for obj in objs:
str_ = labelize(obj)
it = self.hash_ring.iterate_nodes(str_)
# put obj into an mq node.
put_node = next(it)
node_objs[put_node].append(obj)
for _ in xrange(self.copies):
backup_node = next(it)
if backup_node is None: continue
backup_node_objs[backup_node][put_node].append(obj)
return node_objs, backup_node_objs
def remove_node(self, addr):
if addr in self.nodes:
self.nodes.remove(addr)
self.hash_ring = HashRing(self.nodes)
def add_node(self, addr):
if addr not in self.nodes:
self.nodes.append(addr)
self.hash_ring = HashRing(self.nodes)
class Distributor(object):
def __init__(self, addrs, copies=1):
self.nodes = list(addrs)
self.hash_ring = HashRing(self.nodes)
self.copies = min(copies, len(self.nodes)-1)
def distribute(self, objs):
node_objs = defaultdict(list)
backup_node_objs = defaultdict(lambda: defaultdict(list))
if isinstance(objs, basestring) or not iterable(objs):
objs = [objs, ]
for obj in objs:
str_ = labelize(obj)
it = self.hash_ring.iterate_nodes(str_)
# put obj into an mq node.
put_node = next(it)
node_objs[put_node].append(obj)
for _ in xrange(self.copies):
backup_node = next(it)
if backup_node is None: continue
backup_node_objs[backup_node][put_node].append(obj)
return node_objs, backup_node_objs
def remove_node(self, addr):
if addr in self.nodes:
self.nodes.remove(addr)
self.hash_ring = HashRing(self.nodes)
def add_node(self, addr):
if addr not in self.nodes:
self.nodes.append(addr)
self.hash_ring = HashRing(self.nodes)
def __init__(self, nodes, copies=1):
self.nodes = nodes
self.hash_ring = HashRing(self.nodes)
self.copies = max(min(len(self.nodes) - 1, copies), 0)
self.mq = MessageQueue(nodes, copies=copies)
def add_node(self, addr):
if addr not in self.nodes:
self.nodes.append(addr)
self.hash_ring = HashRing(self.nodes)
def remove_node(self, addr):
if addr in self.nodes:
self.nodes.remove(addr)
self.hash_ring = HashRing(self.nodes)
def __init__(self, addrs, copies=1):
self.nodes = list(addrs)
self.hash_ring = HashRing(self.nodes)
self.copies = min(copies, len(self.nodes)-1)
def add_node(self, addr):
if addr not in self.nodes:
self.nodes.append(addr)
self.hash_ring = HashRing(self.nodes)
def __init__(self, addrs, copies=1):
self.nodes = list(addrs)
self.hash_ring = HashRing(self.nodes)
self.copies = min(copies, len(self.nodes) - 1)
class MessageQueue(object):
def __init__(self,
nodes,
local_node=None,
rpc_server=None,
local_store=None,
backup_stores=None,
copies=1):
self.nodes = nodes
self.local_node = local_node
self.local_store = local_store
self.rpc_server = rpc_server
self.backup_stores = backup_stores
self.hash_ring = HashRing(self.nodes)
self.copies = max(min(len(self.nodes) - 1, copies), 0)
if rpc_server is not None and \
self.local_store is not None and \
self.backup_stores is not None:
self._register_rpc()
def _register_rpc(self):
self.rpc_server.register_function(self.put_backup, 'put_backup')
self.rpc_server.register_instance(self.local_store)
def init_store(self,
local_store_path,
backup_stores_path,
verify_exists_hook=None):
self.local_store = Node(local_store_path,
verify_exists_hook=verify_exists_hook)
self.backup_stores_path = backup_stores_path
backup_nodes = self.nodes[:]
backup_nodes.remove(self.local_node)
self.backup_stores = {}
for backup_node in backup_nodes:
backup_node_dir = backup_node.replace(':', '_')
backup_path = os.path.join(backup_stores_path, backup_node_dir)
if not os.path.exists(backup_path):
os.makedirs(backup_path)
self.backup_stores[backup_node] = Node(backup_path,
size=512 * 1024)
self._register_rpc()
def _put(self, node, objs):
if node == self.local_node:
self.local_store.put(objs)
else:
client_call(node, 'put', objs)
def _put_backup(self, node, src, objs):
if node == self.local_node:
self.put_backup(src, objs)
else:
client_call(node, 'put_backup', src, objs)
def _get(self, node):
if node == self.local_node:
return self.local_store.get()
else:
return client_call(node, 'get')
def put(self, obj_or_objs):
def _check(obj):
if not isinstance(obj, str):
raise ValueError("MessageQueue can only put string objects.")
if isinstance(obj_or_objs, (tuple, list)):
for obj in obj_or_objs:
_check(obj)
objs = obj_or_objs
else:
_check(obj_or_objs)
objs = [obj_or_objs]
puts = {}
bkup_puts = {}
for obj in objs:
it = self.hash_ring.iterate_nodes(obj)
# put obj into an mq node.
put_node = next(it)
obs = puts.get(put_node, [])
obs.append(obj)
puts[put_node] = obs
for _ in xrange(self.copies):
bkup_node = next(it)
if bkup_node is None: continue
kv = bkup_puts.get(bkup_node, {})
obs = kv.get(put_node, [])
obs.append(obj)
kv[put_node] = obs
bkup_puts[bkup_node] = kv
for k, v in puts.iteritems():
self._put(k, v)
for k, v in bkup_puts.iteritems():
for src_node, obs in v.iteritems():
self._put_backup(k, src_node, obs)
def put_backup(self, src, obj_or_objs):
backup_store = self.backup_stores[src]
backup_store.put(obj_or_objs)
def get(self):
if self.local_node is not None:
nodes = sorted(self.nodes,
key=lambda k: k == self.local_node,
reverse=True)
else:
nodes = self.nodes
for n in nodes:
obj = self._get(n)
if obj is not None:
return obj
def remove_node(self, node):
self.nodes.remove(node)
self.hash_ring = HashRing(self.nodes)
backup_store = self.backup_stores[node]
obj = backup_store.get()
while obj is not None:
self.local_store.put(obj)
obj = backup_store.get()
backup_store.shutdown()
del self.backup_stores[node]
def add_node(self, node, backup_store=None):
self.nodes.append(node)
self.hash_ring = HashRing(self.nodes)
if backup_store is not None:
self.backup_stores[node] = backup_store
else:
backup_stores_path = getattr(self, 'backup_stores_path')
if backup_stores_path is not None:
path = os.path.join(backup_stores_path, node.replace(':', '_'))
if not os.path.exists(path):
os.makedirs(path)
self.backup_stores[node] = Node(path, size=512 * 1024)
def shutdown(self):
self.local_store.shutdown()
for backup_store in self.backup_stores.values():
backup_store.shutdown()
def __enter__(self):
return self
def __exit__(self):
self.shutdown()
class MessageQueue(object):
def __init__(self, nodes, local_node=None, rpc_server=None,
local_store=None, backup_stores=None, copies=1):
self.nodes = nodes
self.local_node = local_node
self.local_store = local_store
self.rpc_server = rpc_server
self.backup_stores = backup_stores
self.hash_ring = HashRing(self.nodes)
self.copies = max(min(len(self.nodes)-1, copies), 0)
if rpc_server is not None and \
self.local_store is not None and \
self.backup_stores is not None:
self._register_rpc()
def _register_rpc(self):
self.rpc_server.register_function(self.put_backup, 'put_backup')
self.rpc_server.register_instance(self.local_store)
def init_store(self, local_store_path, backup_stores_path,
verify_exists_hook=None):
self.local_store = Node(local_store_path,
verify_exists_hook=verify_exists_hook)
self.backup_stores_path = backup_stores_path
backup_nodes = self.nodes[:]
backup_nodes.remove(self.local_node)
self.backup_stores = {}
for backup_node in backup_nodes:
backup_node_dir = backup_node.replace(':', '_')
backup_path = os.path.join(backup_stores_path, backup_node_dir)
if not os.path.exists(backup_path):
os.makedirs(backup_path)
self.backup_stores[backup_node] = Node(backup_path,
size=512*1024)
self._register_rpc()
def _put(self, node, objs):
if node == self.local_node:
self.local_store.put(objs)
else:
client_call(node, 'put', objs)
def _put_backup(self, node, src, objs):
if node == self.local_node:
self.put_backup(src, objs)
else:
client_call(node, 'put_backup', src, objs)
def _get(self, node):
if node == self.local_node:
return self.local_store.get()
else:
return client_call(node, 'get')
def put(self, obj_or_objs):
def _check(obj):
if not isinstance(obj, str):
raise ValueError("MessageQueue can only put string objects.")
if isinstance(obj_or_objs, (tuple, list)):
for obj in obj_or_objs:
_check(obj)
objs = obj_or_objs
else:
_check(obj_or_objs)
objs = [obj_or_objs]
puts = {}
bkup_puts = {}
for obj in objs:
it = self.hash_ring.iterate_nodes(obj)
# put obj into an mq node.
put_node = next(it)
obs = puts.get(put_node, [])
obs.append(obj)
puts[put_node] = obs
for _ in xrange(self.copies):
bkup_node = next(it)
if bkup_node is None: continue
kv = bkup_puts.get(bkup_node, {})
obs = kv.get(put_node, [])
obs.append(obj)
kv[put_node] = obs
bkup_puts[bkup_node] = kv
for k, v in puts.iteritems():
self._put(k, v)
for k, v in bkup_puts.iteritems():
for src_node, obs in v.iteritems():
self._put_backup(k, src_node, obs)
def put_backup(self, src, obj_or_objs):
backup_store = self.backup_stores[src]
backup_store.put(obj_or_objs)
def get(self):
if self.local_node is not None:
nodes = sorted(self.nodes, key=lambda k: k==self.local_node, reverse=True)
else:
nodes = self.nodes
for n in nodes:
obj = self._get(n)
if obj is not None:
return obj
def remove_node(self, node):
self.nodes.remove(node)
self.hash_ring = HashRing(self.nodes)
backup_store = self.backup_stores[node]
obj = backup_store.get()
while obj is not None:
self.local_store.put(obj)
obj = backup_store.get()
backup_store.shutdown()
del self.backup_stores[node]
def add_node(self, node, backup_store=None):
self.nodes.append(node)
self.hash_ring = HashRing(self.nodes)
if backup_store is not None:
self.backup_stores[node] = backup_store
else:
backup_stores_path = getattr(self, 'backup_stores_path')
if backup_stores_path is not None:
path = os.path.join(backup_stores_path, node.replace(':', '_'))
if not os.path.exists(path):
os.makedirs(path)
self.backup_stores[node] = Node(path,
size=512*1024)
def shutdown(self):
self.local_store.shutdown()
for backup_store in self.backup_stores.values():
backup_store.shutdown()
def __enter__(self):
return self
def __exit__(self):
self.shutdown()
def remove_node(self, addr):
if addr in self.nodes:
self.nodes.remove(addr)
self.hash_ring = HashRing(self.nodes)
