def testBatchPutGet(self):
self.node.shutdown()
size = 60
# '1'*20 will spend 30 bytes in `mq store`
batch1 = ['1' * 20, '2' * 20]
batch2 = ['3' * 20, '4' * 20]
self.node = Store(self.dir_, size)
self.assertEqual(self.node.put(batch1), batch1)
self.assertEqual(self.node.put(batch2), batch2)
self.assertEqual(len(self.node.legal_files), 2)
gets = sorted([self.node.get() for _ in range(4)])
res = list(batch1)
res.extend(batch2)
self.assertEqual(gets, res)
self.node.put('5' * 20)
self.assertEqual(self.node.get(), '5' * 20)
self.node.put('6' * 20)
self.assertEqual(len(self.node.legal_files), 1)
self.assertEqual(self.node.get(), '6' * 20)
self.assertEqual(self.node.get(), None)
self.assertRaises(StoreNoSpaceForPut, lambda: self.node.put('7' * 100))
def add_node(self, addr):
if addr in self.addrs: return
self.addrs.append(addr)
backup_store_dir = os.path.join(self.dir_, BACKUP_STORE_FN)
backup_node_dir = addr.replace(':', '_')
backup_path = os.path.join(backup_store_dir, backup_node_dir)
self.backup_stores[addr] = Store(backup_path,
size=512*1024, mkdirs=True)
def add_node(self, addr):
"""
When a new message queue node is in, firstly will add the address
to the known queue nodes, then a backup for this node will be created.
"""
if addr in self.addrs: return
self.addrs.append(addr)
backup_store_dir = os.path.join(self.dir_, BACKUP_STORE_FN)
backup_node_dir = addr.replace(':', '_')
backup_path = os.path.join(backup_store_dir, backup_node_dir)
self.backup_stores[addr] = Store(backup_path,
size=512*1024, mkdirs=True)
def init(self):
with self._lock:
if self.inited: return
get_priority_store_dir = lambda priority: os.path.join(self.dir_,
PRIORITY_STORE_FN, str(priority))
self.priority_stores = [Store(get_priority_store_dir(i),
deduper=self.deduper,
mkdirs=True) \
for i in range(self.n_priorities)]
backup_store_dir = os.path.join(self.dir_, BACKUP_STORE_FN)
self.backup_stores = {}
for backup_addr in self.other_addrs:
backup_node_dir = backup_addr.replace(':', '_')
backup_path = os.path.join(backup_store_dir, backup_node_dir)
self.backup_stores[backup_addr] = Store(backup_path,
size=512*1024, mkdirs=True)
inc_store_dir = os.path.join(self.dir_, INCR_STORE_FN)
self.inc_store = Store(inc_store_dir, mkdirs=True)
self.inited = True
def testPutGet(self):
num = str(12345)
self.assertEqual(self.store.put(num), num)
self.assertEqual(self.store.put(num), None)
num2 = str(67890)
nums = [num, num2]
self.assertEqual(self.store.put(nums), [num2])
self.store.shutdown()
self.store.deduper.shutdown()
self.assertGreater(
os.path.getsize(os.path.join(self.dir_, 'bloomfilter')), 0)
bloom_filter_deduper = FileBloomFilterDeduper(
os.path.join(self.dir_, 'bloomfilter'), 5)
self.store = Store(self.node_dir, deduper=bloom_filter_deduper)
num3 = str(13579)
nums = [num, num2, num3]
self.assertEqual(self.store.put(nums), [num3])
def testPutCloseGet(self):
self.node.put('1' * 10)
self.node.shutdown()
self.node = Store(self.dir_)
self.assertEqual(self.node.get(), '1' * 10)
def setUp(self):
self.dir_ = tempfile.mkdtemp()
self.node = Store(self.dir_)
def setUp(self):
self.dir_ = tempfile.mkdtemp()
self.node_dir = os.path.join(self.dir_, 'node')
deduper = FileBloomFilterDeduper(
os.path.join(self.dir_, 'bloomfilter'), 10)
self.store = Store(self.node_dir, deduper=deduper, mkdirs=True)
