def distribute(self, objs):
"""
:param objs: the objects
:return: a tuple with two dicts,
the first is the mapping from the node address to the belonged objects,
the second is the mapping from the backup node address and the objects
"""
node_objs = defaultdict(list)
backup_node_objs = defaultdict(lambda: defaultdict(list))
if isinstance(objs, basestring) or not iterable(objs):
objs = [objs, ]
for obj in objs:
str_ = labelize(obj)
it = self.hash_ring.iterate_nodes(str_)
# put obj into an mq node.
put_node = next(it)
node_objs[put_node].append(obj)
for _ in xrange(self.copies):
backup_node = next(it)
if backup_node is None: continue
backup_node_objs[backup_node][put_node].append(obj)
return node_objs, backup_node_objs
def distribute(self, objs):
node_objs = defaultdict(list)
backup_node_objs = defaultdict(lambda: defaultdict(list))
if isinstance(objs, basestring) or not iterable(objs):
objs = [
objs,
]
for obj in objs:
str_ = labelize(obj)
it = self.hash_ring.iterate_nodes(str_)
# put obj into an mq node.
put_node = next(it)
node_objs[put_node].append(obj)
for _ in xrange(self.copies):
backup_node = next(it)
if backup_node is None: continue
backup_node_objs[backup_node][put_node].append(obj)
return node_objs, backup_node_objs
def put(self, objects, force=False, commit=True):
if self.stopped: return
self.init()
if isinstance(objects, basestring) or not iterable(objects):
return self.put_one(objects, force, commit)
remains = []
for obj in objects:
result = self.put_one(obj, force, commit=False)
if result is not None:
remains.append(result)
m = self.map_handles[WRITE_ENTRANCE]
if len(remains) > 0 and m is not None:
with self.lock:
m.flush()
return remains
def put(self, objects, force=False, commit=True):
if self.stopped: return
self.init()
if isinstance(objects, basestring) or not iterable(objects):
return self.put_one(objects, force, commit)
remains = []
for obj in objects:
result = self.put_one(obj, force, commit=False)
if result is not None:
remains.append(result)
m = self.map_handles[WRITE_ENTRANCE]
if len(remains) > 0 and m is not None:
with self.lock:
m.flush()
return remains
def distribute(self, objs):
node_objs = defaultdict(list)
backup_node_objs = defaultdict(lambda: defaultdict(list))
if isinstance(objs, basestring) or not iterable(objs):
objs = [objs, ]
for obj in objs:
str_ = labelize(obj)
it = self.hash_ring.iterate_nodes(str_)
# put obj into an mq node.
put_node = next(it)
node_objs[put_node].append(obj)
for _ in xrange(self.copies):
backup_node = next(it)
if backup_node is None: continue
backup_node_objs[backup_node][put_node].append(obj)
return node_objs, backup_node_objs
def create_combiner(self, val):
if not iterable(val):
val = [val]
return set(val)
def create_combiner(self, val):
if not iterable(val):
val = [val]
return set(val)