def distribute(self, objs):
node_objs = defaultdict(list)
backup_node_objs = defaultdict(lambda: defaultdict(list))
if isinstance(objs, basestring) or not iterable(objs):
objs = [
objs,
]
for obj in objs:
str_ = labelize(obj)
it = self.hash_ring.iterate_nodes(str_)
# put obj into an mq node.
put_node = next(it)
node_objs[put_node].append(obj)
for _ in xrange(self.copies):
backup_node = next(it)
if backup_node is None: continue
backup_node_objs[backup_node][put_node].append(obj)
return node_objs, backup_node_objs
def distribute(self, objs):
"""
:param objs: the objects
:return: a tuple with two dicts,
the first is the mapping from the node address to the belonged objects,
the second is the mapping from the backup node address and the objects
"""
node_objs = defaultdict(list)
backup_node_objs = defaultdict(lambda: defaultdict(list))
if isinstance(objs, basestring) or not iterable(objs):
objs = [objs, ]
for obj in objs:
str_ = labelize(obj)
it = self.hash_ring.iterate_nodes(str_)
# put obj into an mq node.
put_node = next(it)
node_objs[put_node].append(obj)
for _ in xrange(self.copies):
backup_node = next(it)
if backup_node is None: continue
backup_node_objs[backup_node][put_node].append(obj)
return node_objs, backup_node_objs
def _init_process(self, agent):
while not self.stopped.is_set():
need_process = agent.poll(10)
if self.stopped.is_set():
return
if not need_process:
continue
action, data = agent.recv()
if action == PUT:
objs, flush = data
self.put(objs, flush=flush)
agent.send(1)
elif action == PUT_INC:
self.put_inc(data)
agent.send(1)
elif action == GET:
size, priority = data
agent.send(self.get(size=size,
priority=priority))
elif action == GET_INC:
agent.send(self.get_inc(data))
elif action == EXIST:
if not self.mq_node.deduper:
agent.send(False)
else:
agent.send(self.exist(labelize(data)))
else:
raise ValueError('mq client can only put, put_inc, and get')
def put_one(self, obj, force=False, commit=True):
"""
Put one object into the storage.
:param obj: the object to put into
:param force: if True, the deduper will not check if the obj has been
put into the entire message queue before
:param commit: if True, the mmap will be forced to flush
"""
if self.stopped: return
# self.init()
if isinstance(obj, str) and obj.strip() == '':
return
if not force and self.deduper is not None:
prop = labelize(obj)
if self.deduper.exist(prop):
return
if len(self.legal_files) == 0:
with self.lock:
self._generate_file()
obj_str = self._stringfy(obj)
# If no file has enough space
if len(obj_str) + 4 > self.store_file_size:
raise StoreNoSpaceForPut('No enouph space for this put.')
with self.lock:
m = self.map_handles[WRITE_ENTRANCE]
pos = self._seek_writable_pos(m)
size = pos + 4 + len(obj_str)
if pos < 0 or size > self.store_file_size:
m.flush()
self._generate_file()
pos = 0
size = 4 + len(obj_str)
m = self.map_handles[WRITE_ENTRANCE]
m[:size] = m[:pos] + struct.pack('I', len(obj_str)) + obj_str
if commit is True:
m.flush()
return obj
def put_one(self, obj, force=False, commit=True):
"""
Put one object into the storage.
:param obj: the object to put into
:param force: if True, the deduper will not check if the obj has been
put into the entire message queue before
:param commit: if True, the mmap will be forced to flush
"""
if self.stopped: return
# self.init()
if isinstance(obj, str) and obj.strip() == '':
return
if not force and self.deduper is not None:
prop = labelize(obj)
if self.deduper.exist(prop):
return
if len(self.legal_files) == 0:
self._generate_file()
obj_str = self._stringfy(obj)
# If no file has enough space
if len(obj_str) + 4 > self.store_file_size:
raise StoreNoSpaceForPut('No enouph space for this put.')
with self.lock:
m = self.map_handles[WRITE_ENTRANCE]
pos = self._seek_writable_pos(m)
size = pos + 4 + len(obj_str)
if pos < 0 or size > self.store_file_size:
m.flush()
self._generate_file()
pos = 0
size = 4 + len(obj_str)
m = self.map_handles[WRITE_ENTRANCE]
m[:size] = m[:pos] + struct.pack('I', len(obj_str)) + obj_str
if commit is True:
m.flush()
return obj
def distribute(self, objs):
node_objs = defaultdict(list)
backup_node_objs = defaultdict(lambda: defaultdict(list))
if isinstance(objs, basestring) or not iterable(objs):
objs = [objs, ]
for obj in objs:
str_ = labelize(obj)
it = self.hash_ring.iterate_nodes(str_)
# put obj into an mq node.
put_node = next(it)
node_objs[put_node].append(obj)
for _ in xrange(self.copies):
backup_node = next(it)
if backup_node is None: continue
backup_node_objs[backup_node][put_node].append(obj)
return node_objs, backup_node_objs
def put_one(self, obj, force=False, commit=True):
if self.stopped:
return
# self.init()
if isinstance(obj, str) and obj.strip() == "":
return
if not force and self.deduper is not None:
prop = labelize(obj)
if self.deduper.exist(prop):
return
if len(self.legal_files) == 0:
self._generate_file()
obj_str = self._stringfy(obj)
# If no file has enough space
if len(obj_str) + 4 > self.store_file_size:
raise StoreNoSpaceForPut("No enouph space for this put.")
with self.lock:
m = self.map_handles[WRITE_ENTRANCE]
pos = self._seek_writable_pos(m)
size = pos + 4 + len(obj_str)
if pos < 0 or size > self.store_file_size:
m.flush()
self._generate_file()
pos = 0
size = 4 + len(obj_str)
m = self.map_handles[WRITE_ENTRANCE]
m[:size] = m[:pos] + struct.pack("I", len(obj_str)) + obj_str
if commit is True:
m.flush()
return obj
def put_one(self, obj, force=False, commit=True):
if self.stopped: return
# self.init()
if isinstance(obj, str) and obj.strip() == '':
return
if not force and self.deduper is not None:
prop = labelize(obj)
if self.deduper.exist(prop):
return
if len(self.legal_files) == 0:
self._generate_file()
obj_str = self._stringfy(obj)
# If no file has enough space
if len(obj_str) + 4 > self.store_file_size:
raise StoreNoSpaceForPut('No enouph space for this put.')
with self.lock:
m = self.map_handles[WRITE_ENTRANCE]
pos = self._seek_writable_pos(m)
size = pos + 4 + len(obj_str)
if pos < 0 or size > self.store_file_size:
m.flush()
self._generate_file()
pos = 0
size = 4 + len(obj_str)
m = self.map_handles[WRITE_ENTRANCE]
m[:size] = m[:pos] + struct.pack('I', len(obj_str)) + obj_str
if commit is True:
m.flush()
return obj
def exist(self, obj):
self.conn.send((EXIST, labelize(obj)))
return self.conn.recv()
def exist(self, obj):
if self.deduper:
return self.deduper.exist(labelize(obj))
return False
