def distribute(self, objs): node_objs = defaultdict(list) backup_node_objs = defaultdict(lambda: defaultdict(list)) if isinstance(objs, basestring) or not iterable(objs): objs = [ objs, ] for obj in objs: str_ = labelize(obj) it = self.hash_ring.iterate_nodes(str_) # put obj into an mq node. put_node = next(it) node_objs[put_node].append(obj) for _ in xrange(self.copies): backup_node = next(it) if backup_node is None: continue backup_node_objs[backup_node][put_node].append(obj) return node_objs, backup_node_objs
def distribute(self, objs): """ :param objs: the objects :return: a tuple with two dicts, the first is the mapping from the node address to the belonged objects, the second is the mapping from the backup node address and the objects """ node_objs = defaultdict(list) backup_node_objs = defaultdict(lambda: defaultdict(list)) if isinstance(objs, basestring) or not iterable(objs): objs = [objs, ] for obj in objs: str_ = labelize(obj) it = self.hash_ring.iterate_nodes(str_) # put obj into an mq node. put_node = next(it) node_objs[put_node].append(obj) for _ in xrange(self.copies): backup_node = next(it) if backup_node is None: continue backup_node_objs[backup_node][put_node].append(obj) return node_objs, backup_node_objs
def _init_process(self, agent): while not self.stopped.is_set(): need_process = agent.poll(10) if self.stopped.is_set(): return if not need_process: continue action, data = agent.recv() if action == PUT: objs, flush = data self.put(objs, flush=flush) agent.send(1) elif action == PUT_INC: self.put_inc(data) agent.send(1) elif action == GET: size, priority = data agent.send(self.get(size=size, priority=priority)) elif action == GET_INC: agent.send(self.get_inc(data)) elif action == EXIST: if not self.mq_node.deduper: agent.send(False) else: agent.send(self.exist(labelize(data))) else: raise ValueError('mq client can only put, put_inc, and get')
def put_one(self, obj, force=False, commit=True): """ Put one object into the storage. :param obj: the object to put into :param force: if True, the deduper will not check if the obj has been put into the entire message queue before :param commit: if True, the mmap will be forced to flush """ if self.stopped: return # self.init() if isinstance(obj, str) and obj.strip() == '': return if not force and self.deduper is not None: prop = labelize(obj) if self.deduper.exist(prop): return if len(self.legal_files) == 0: with self.lock: self._generate_file() obj_str = self._stringfy(obj) # If no file has enough space if len(obj_str) + 4 > self.store_file_size: raise StoreNoSpaceForPut('No enouph space for this put.') with self.lock: m = self.map_handles[WRITE_ENTRANCE] pos = self._seek_writable_pos(m) size = pos + 4 + len(obj_str) if pos < 0 or size > self.store_file_size: m.flush() self._generate_file() pos = 0 size = 4 + len(obj_str) m = self.map_handles[WRITE_ENTRANCE] m[:size] = m[:pos] + struct.pack('I', len(obj_str)) + obj_str if commit is True: m.flush() return obj
def put_one(self, obj, force=False, commit=True): """ Put one object into the storage. :param obj: the object to put into :param force: if True, the deduper will not check if the obj has been put into the entire message queue before :param commit: if True, the mmap will be forced to flush """ if self.stopped: return # self.init() if isinstance(obj, str) and obj.strip() == '': return if not force and self.deduper is not None: prop = labelize(obj) if self.deduper.exist(prop): return if len(self.legal_files) == 0: self._generate_file() obj_str = self._stringfy(obj) # If no file has enough space if len(obj_str) + 4 > self.store_file_size: raise StoreNoSpaceForPut('No enouph space for this put.') with self.lock: m = self.map_handles[WRITE_ENTRANCE] pos = self._seek_writable_pos(m) size = pos + 4 + len(obj_str) if pos < 0 or size > self.store_file_size: m.flush() self._generate_file() pos = 0 size = 4 + len(obj_str) m = self.map_handles[WRITE_ENTRANCE] m[:size] = m[:pos] + struct.pack('I', len(obj_str)) + obj_str if commit is True: m.flush() return obj
def distribute(self, objs): node_objs = defaultdict(list) backup_node_objs = defaultdict(lambda: defaultdict(list)) if isinstance(objs, basestring) or not iterable(objs): objs = [objs, ] for obj in objs: str_ = labelize(obj) it = self.hash_ring.iterate_nodes(str_) # put obj into an mq node. put_node = next(it) node_objs[put_node].append(obj) for _ in xrange(self.copies): backup_node = next(it) if backup_node is None: continue backup_node_objs[backup_node][put_node].append(obj) return node_objs, backup_node_objs
def put_one(self, obj, force=False, commit=True): if self.stopped: return # self.init() if isinstance(obj, str) and obj.strip() == "": return if not force and self.deduper is not None: prop = labelize(obj) if self.deduper.exist(prop): return if len(self.legal_files) == 0: self._generate_file() obj_str = self._stringfy(obj) # If no file has enough space if len(obj_str) + 4 > self.store_file_size: raise StoreNoSpaceForPut("No enouph space for this put.") with self.lock: m = self.map_handles[WRITE_ENTRANCE] pos = self._seek_writable_pos(m) size = pos + 4 + len(obj_str) if pos < 0 or size > self.store_file_size: m.flush() self._generate_file() pos = 0 size = 4 + len(obj_str) m = self.map_handles[WRITE_ENTRANCE] m[:size] = m[:pos] + struct.pack("I", len(obj_str)) + obj_str if commit is True: m.flush() return obj
def put_one(self, obj, force=False, commit=True): if self.stopped: return # self.init() if isinstance(obj, str) and obj.strip() == '': return if not force and self.deduper is not None: prop = labelize(obj) if self.deduper.exist(prop): return if len(self.legal_files) == 0: self._generate_file() obj_str = self._stringfy(obj) # If no file has enough space if len(obj_str) + 4 > self.store_file_size: raise StoreNoSpaceForPut('No enouph space for this put.') with self.lock: m = self.map_handles[WRITE_ENTRANCE] pos = self._seek_writable_pos(m) size = pos + 4 + len(obj_str) if pos < 0 or size > self.store_file_size: m.flush() self._generate_file() pos = 0 size = 4 + len(obj_str) m = self.map_handles[WRITE_ENTRANCE] m[:size] = m[:pos] + struct.pack('I', len(obj_str)) + obj_str if commit is True: m.flush() return obj
def exist(self, obj): self.conn.send((EXIST, labelize(obj))) return self.conn.recv()
def exist(self, obj): if self.deduper: return self.deduper.exist(labelize(obj)) return False