def add_node(self, node, backup_store=None): self.nodes.append(node) self.hash_ring = HashRing(self.nodes) if backup_store is not None: self.backup_stores[node] = backup_store else: backup_stores_path = getattr(self, 'backup_stores_path') if backup_stores_path is not None: path = os.path.join(backup_stores_path, node.replace(':', '_')) if not os.path.exists(path): os.makedirs(path) self.backup_stores[node] = Node(path, size=512 * 1024)
def remove_node(self, node): self.nodes.remove(node) self.hash_ring = HashRing(self.nodes) backup_store = self.backup_stores[node] obj = backup_store.get() while obj is not None: self.local_store.put(obj) obj = backup_store.get() backup_store.shutdown() del self.backup_stores[node]
def __init__(self, nodes, local_node=None, rpc_server=None, local_store=None, backup_stores=None, copies=1): self.nodes = nodes self.local_node = local_node self.local_store = local_store self.rpc_server = rpc_server self.backup_stores = backup_stores self.hash_ring = HashRing(self.nodes) self.copies = max(min(len(self.nodes) - 1, copies), 0) if rpc_server is not None and \ self.local_store is not None and \ self.backup_stores is not None: self._register_rpc()
def __init__(self, nodes, local_node=None, rpc_server=None, local_store=None, backup_stores=None, copies=1): self.nodes = nodes self.local_node = local_node self.local_store = local_store self.rpc_server = rpc_server self.backup_stores = backup_stores self.hash_ring = HashRing(self.nodes) self.copies = max(min(len(self.nodes) - 1, copies), 0) if rpc_server is not None and self.local_store is not None and self.backup_stores is not None: self._register_rpc()
def add_node(self, node, backup_store=None): self.nodes.append(node) self.hash_ring = HashRing(self.nodes) if backup_store is not None: self.backup_stores[node] = backup_store else: backup_stores_path = getattr(self, "backup_stores_path") if backup_stores_path is not None: path = os.path.join(backup_stores_path, node.replace(":", "_")) if not os.path.exists(path): os.makedirs(path) self.backup_stores[node] = Node(path, size=512 * 1024)
class Distributor(object): """ Given several objects, to decides which message queue node each one belong to. """ def __init__(self, addrs, copies=1): self.nodes = list(addrs) self.hash_ring = HashRing(self.nodes) self.copies = min(copies, len(self.nodes)-1) def distribute(self, objs): """ :param objs: the objects :return: a tuple with two dicts, the first is the mapping from the node address to the belonged objects, the second is the mapping from the backup node address and the objects """ node_objs = defaultdict(list) backup_node_objs = defaultdict(lambda: defaultdict(list)) if isinstance(objs, basestring) or not iterable(objs): objs = [objs, ] for obj in objs: str_ = labelize(obj) it = self.hash_ring.iterate_nodes(str_) # put obj into an mq node. put_node = next(it) node_objs[put_node].append(obj) for _ in xrange(self.copies): backup_node = next(it) if backup_node is None: continue backup_node_objs[backup_node][put_node].append(obj) return node_objs, backup_node_objs def remove_node(self, addr): if addr in self.nodes: self.nodes.remove(addr) self.hash_ring = HashRing(self.nodes) def add_node(self, addr): if addr not in self.nodes: self.nodes.append(addr) self.hash_ring = HashRing(self.nodes)
class Distributor(object): def __init__(self, addrs, copies=1): self.nodes = list(addrs) self.hash_ring = HashRing(self.nodes) self.copies = min(copies, len(self.nodes) - 1) def distribute(self, objs): node_objs = defaultdict(list) backup_node_objs = defaultdict(lambda: defaultdict(list)) if isinstance(objs, basestring) or not iterable(objs): objs = [ objs, ] for obj in objs: str_ = labelize(obj) it = self.hash_ring.iterate_nodes(str_) # put obj into an mq node. put_node = next(it) node_objs[put_node].append(obj) for _ in xrange(self.copies): backup_node = next(it) if backup_node is None: continue backup_node_objs[backup_node][put_node].append(obj) return node_objs, backup_node_objs def remove_node(self, addr): if addr in self.nodes: self.nodes.remove(addr) self.hash_ring = HashRing(self.nodes) def add_node(self, addr): if addr not in self.nodes: self.nodes.append(addr) self.hash_ring = HashRing(self.nodes)
class Distributor(object): def __init__(self, addrs, copies=1): self.nodes = list(addrs) self.hash_ring = HashRing(self.nodes) self.copies = min(copies, len(self.nodes)-1) def distribute(self, objs): node_objs = defaultdict(list) backup_node_objs = defaultdict(lambda: defaultdict(list)) if isinstance(objs, basestring) or not iterable(objs): objs = [objs, ] for obj in objs: str_ = labelize(obj) it = self.hash_ring.iterate_nodes(str_) # put obj into an mq node. put_node = next(it) node_objs[put_node].append(obj) for _ in xrange(self.copies): backup_node = next(it) if backup_node is None: continue backup_node_objs[backup_node][put_node].append(obj) return node_objs, backup_node_objs def remove_node(self, addr): if addr in self.nodes: self.nodes.remove(addr) self.hash_ring = HashRing(self.nodes) def add_node(self, addr): if addr not in self.nodes: self.nodes.append(addr) self.hash_ring = HashRing(self.nodes)
def __init__(self, nodes, copies=1): self.nodes = nodes self.hash_ring = HashRing(self.nodes) self.copies = max(min(len(self.nodes) - 1, copies), 0) self.mq = MessageQueue(nodes, copies=copies)
def add_node(self, addr): if addr not in self.nodes: self.nodes.append(addr) self.hash_ring = HashRing(self.nodes)
def remove_node(self, addr): if addr in self.nodes: self.nodes.remove(addr) self.hash_ring = HashRing(self.nodes)
def __init__(self, addrs, copies=1): self.nodes = list(addrs) self.hash_ring = HashRing(self.nodes) self.copies = min(copies, len(self.nodes)-1)
def __init__(self, addrs, copies=1): self.nodes = list(addrs) self.hash_ring = HashRing(self.nodes) self.copies = min(copies, len(self.nodes) - 1)
class MessageQueue(object): def __init__(self, nodes, local_node=None, rpc_server=None, local_store=None, backup_stores=None, copies=1): self.nodes = nodes self.local_node = local_node self.local_store = local_store self.rpc_server = rpc_server self.backup_stores = backup_stores self.hash_ring = HashRing(self.nodes) self.copies = max(min(len(self.nodes) - 1, copies), 0) if rpc_server is not None and \ self.local_store is not None and \ self.backup_stores is not None: self._register_rpc() def _register_rpc(self): self.rpc_server.register_function(self.put_backup, 'put_backup') self.rpc_server.register_instance(self.local_store) def init_store(self, local_store_path, backup_stores_path, verify_exists_hook=None): self.local_store = Node(local_store_path, verify_exists_hook=verify_exists_hook) self.backup_stores_path = backup_stores_path backup_nodes = self.nodes[:] backup_nodes.remove(self.local_node) self.backup_stores = {} for backup_node in backup_nodes: backup_node_dir = backup_node.replace(':', '_') backup_path = os.path.join(backup_stores_path, backup_node_dir) if not os.path.exists(backup_path): os.makedirs(backup_path) self.backup_stores[backup_node] = Node(backup_path, size=512 * 1024) self._register_rpc() def _put(self, node, objs): if node == self.local_node: self.local_store.put(objs) else: client_call(node, 'put', objs) def _put_backup(self, node, src, objs): if node == self.local_node: self.put_backup(src, objs) else: client_call(node, 'put_backup', src, objs) def _get(self, node): if node == self.local_node: return self.local_store.get() else: return client_call(node, 'get') def put(self, obj_or_objs): def _check(obj): if not isinstance(obj, str): raise ValueError("MessageQueue can only put string objects.") if isinstance(obj_or_objs, (tuple, list)): for obj in obj_or_objs: _check(obj) objs = obj_or_objs else: _check(obj_or_objs) objs = [obj_or_objs] puts = {} bkup_puts = {} for obj in objs: it = self.hash_ring.iterate_nodes(obj) # put obj into an mq node. put_node = next(it) obs = puts.get(put_node, []) obs.append(obj) puts[put_node] = obs for _ in xrange(self.copies): bkup_node = next(it) if bkup_node is None: continue kv = bkup_puts.get(bkup_node, {}) obs = kv.get(put_node, []) obs.append(obj) kv[put_node] = obs bkup_puts[bkup_node] = kv for k, v in puts.iteritems(): self._put(k, v) for k, v in bkup_puts.iteritems(): for src_node, obs in v.iteritems(): self._put_backup(k, src_node, obs) def put_backup(self, src, obj_or_objs): backup_store = self.backup_stores[src] backup_store.put(obj_or_objs) def get(self): if self.local_node is not None: nodes = sorted(self.nodes, key=lambda k: k == self.local_node, reverse=True) else: nodes = self.nodes for n in nodes: obj = self._get(n) if obj is not None: return obj def remove_node(self, node): self.nodes.remove(node) self.hash_ring = HashRing(self.nodes) backup_store = self.backup_stores[node] obj = backup_store.get() while obj is not None: self.local_store.put(obj) obj = backup_store.get() backup_store.shutdown() del self.backup_stores[node] def add_node(self, node, backup_store=None): self.nodes.append(node) self.hash_ring = HashRing(self.nodes) if backup_store is not None: self.backup_stores[node] = backup_store else: backup_stores_path = getattr(self, 'backup_stores_path') if backup_stores_path is not None: path = os.path.join(backup_stores_path, node.replace(':', '_')) if not os.path.exists(path): os.makedirs(path) self.backup_stores[node] = Node(path, size=512 * 1024) def shutdown(self): self.local_store.shutdown() for backup_store in self.backup_stores.values(): backup_store.shutdown() def __enter__(self): return self def __exit__(self): self.shutdown()
class MessageQueue(object): def __init__(self, nodes, local_node=None, rpc_server=None, local_store=None, backup_stores=None, copies=1): self.nodes = nodes self.local_node = local_node self.local_store = local_store self.rpc_server = rpc_server self.backup_stores = backup_stores self.hash_ring = HashRing(self.nodes) self.copies = max(min(len(self.nodes)-1, copies), 0) if rpc_server is not None and \ self.local_store is not None and \ self.backup_stores is not None: self._register_rpc() def _register_rpc(self): self.rpc_server.register_function(self.put_backup, 'put_backup') self.rpc_server.register_instance(self.local_store) def init_store(self, local_store_path, backup_stores_path, verify_exists_hook=None): self.local_store = Node(local_store_path, verify_exists_hook=verify_exists_hook) self.backup_stores_path = backup_stores_path backup_nodes = self.nodes[:] backup_nodes.remove(self.local_node) self.backup_stores = {} for backup_node in backup_nodes: backup_node_dir = backup_node.replace(':', '_') backup_path = os.path.join(backup_stores_path, backup_node_dir) if not os.path.exists(backup_path): os.makedirs(backup_path) self.backup_stores[backup_node] = Node(backup_path, size=512*1024) self._register_rpc() def _put(self, node, objs): if node == self.local_node: self.local_store.put(objs) else: client_call(node, 'put', objs) def _put_backup(self, node, src, objs): if node == self.local_node: self.put_backup(src, objs) else: client_call(node, 'put_backup', src, objs) def _get(self, node): if node == self.local_node: return self.local_store.get() else: return client_call(node, 'get') def put(self, obj_or_objs): def _check(obj): if not isinstance(obj, str): raise ValueError("MessageQueue can only put string objects.") if isinstance(obj_or_objs, (tuple, list)): for obj in obj_or_objs: _check(obj) objs = obj_or_objs else: _check(obj_or_objs) objs = [obj_or_objs] puts = {} bkup_puts = {} for obj in objs: it = self.hash_ring.iterate_nodes(obj) # put obj into an mq node. put_node = next(it) obs = puts.get(put_node, []) obs.append(obj) puts[put_node] = obs for _ in xrange(self.copies): bkup_node = next(it) if bkup_node is None: continue kv = bkup_puts.get(bkup_node, {}) obs = kv.get(put_node, []) obs.append(obj) kv[put_node] = obs bkup_puts[bkup_node] = kv for k, v in puts.iteritems(): self._put(k, v) for k, v in bkup_puts.iteritems(): for src_node, obs in v.iteritems(): self._put_backup(k, src_node, obs) def put_backup(self, src, obj_or_objs): backup_store = self.backup_stores[src] backup_store.put(obj_or_objs) def get(self): if self.local_node is not None: nodes = sorted(self.nodes, key=lambda k: k==self.local_node, reverse=True) else: nodes = self.nodes for n in nodes: obj = self._get(n) if obj is not None: return obj def remove_node(self, node): self.nodes.remove(node) self.hash_ring = HashRing(self.nodes) backup_store = self.backup_stores[node] obj = backup_store.get() while obj is not None: self.local_store.put(obj) obj = backup_store.get() backup_store.shutdown() del self.backup_stores[node] def add_node(self, node, backup_store=None): self.nodes.append(node) self.hash_ring = HashRing(self.nodes) if backup_store is not None: self.backup_stores[node] = backup_store else: backup_stores_path = getattr(self, 'backup_stores_path') if backup_stores_path is not None: path = os.path.join(backup_stores_path, node.replace(':', '_')) if not os.path.exists(path): os.makedirs(path) self.backup_stores[node] = Node(path, size=512*1024) def shutdown(self): self.local_store.shutdown() for backup_store in self.backup_stores.values(): backup_store.shutdown() def __enter__(self): return self def __exit__(self): self.shutdown()