コード例 #1
0
ファイル: test_bloom_filter_mq.py プロジェクト: zouxc-zz/cola
 def setUp(self):
     self.dir_ = tempfile.mkdtemp()
     self.node_dir = os.path.join(self.dir_, 'node')
     os.mkdir(self.node_dir)
     bloom_filter_hook = FileBloomFilter(
         os.path.join(self.dir_, 'bloomfilter'), 10)
     self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)
コード例 #2
0
ファイル: test_mq_node.py プロジェクト: zouxc-zz/cola
    def testBatchPutGet(self):
        self.node.shutdown()

        size = 50
        batch1 = ['1' * 20, '2' * 20]
        batch2 = ['3' * 20, '4' * 20]

        self.node = Node(self.dir_, size)

        self.assertEqual(self.node.put(batch1), batch1)
        self.assertEqual(self.node.put(batch2), batch2)

        self.assertEqual(len(self.node.map_files), 2)

        gets = sorted([self.node.get() for _ in range(4)])
        res = batch1
        res.extend(batch2)
        self.assertEqual(gets, res)

        self.node.put('5' * 20)
        self.assertEqual(self.node.get(), '5' * 20)

        self.node.put('6' * 20)

        self.node.merge()
        self.assertEqual(len(self.node.map_files), 1)

        self.assertEqual(self.node.get(), '6' * 20)
        self.assertEqual(self.node.get(), None)

        self.assertRaises(NodeNoSpaceForPut, lambda: self.node.put('7' * 100))
コード例 #3
0
ファイル: test_mq_node.py プロジェクト: 0pengl/cola
 def testBatchPutGet(self):
     self.node.shutdown()
     
     size = 50
     batch1 = ['1' * 20, '2' * 20]
     batch2 = ['3' * 20, '4' * 20]
     
     self.node = Node(self.dir_, size)
     
     self.assertEqual(self.node.put(batch1), batch1)
     self.assertEqual(self.node.put(batch2), batch2)
     
     self.assertEqual(len(self.node.map_files), 2)
     
     gets = sorted([self.node.get() for _ in range(4)])
     res = batch1
     res.extend(batch2)
     self.assertEqual(gets, res)
     
     self.node.put('5' * 20)
     self.assertEqual(self.node.get(), '5' * 20)
     
     self.node.put('6' * 20)
     
     self.node.merge()
     self.assertEqual(len(self.node.map_files), 1)
     
     self.assertEqual(self.node.get(), '6' * 20)
     self.assertEqual(self.node.get(), None)
     
     self.assertRaises(NodeNoSpaceForPut, lambda: self.node.put('7' * 100))
コード例 #4
0
ファイル: test_bloom_filter_mq.py プロジェクト: 0pengl/cola
 def setUp(self):
     self.dir_ = tempfile.mkdtemp()
     self.node_dir = os.path.join(self.dir_, 'node')
     os.mkdir(self.node_dir)
     bloom_filter_hook = FileBloomFilter(
         os.path.join(self.dir_, 'bloomfilter'), 10
     )
     self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)
コード例 #5
0
ファイル: test_bloom_filter_mq.py プロジェクト: zouxc-zz/cola
class Test(unittest.TestCase):
    def setUp(self):
        self.dir_ = tempfile.mkdtemp()
        self.node_dir = os.path.join(self.dir_, 'node')
        os.mkdir(self.node_dir)
        bloom_filter_hook = FileBloomFilter(
            os.path.join(self.dir_, 'bloomfilter'), 10)
        self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)

    def tearDown(self):
        self.node.shutdown()
        shutil.rmtree(self.dir_)

    def testPutGet(self):
        num = str(12345)

        self.assertEqual(self.node.put(num), num)
        self.assertEqual(self.node.put(num), '')

        num2 = str(67890)
        nums = [num, num2]
        self.assertEqual(self.node.put(nums), [num2])

        self.node.shutdown()
        self.assertGreater(
            os.path.getsize(os.path.join(self.dir_, 'bloomfilter')), 0)

        bloom_filter_hook = FileBloomFilter(
            os.path.join(self.dir_, 'bloomfilter'), 5)
        self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)

        num3 = str(13579)
        nums = [num, num2, num3]
        self.assertEqual(self.node.put(nums), [num3])
コード例 #6
0
    def init_store(self,
                   local_store_path,
                   backup_stores_path,
                   verify_exists_hook=None):
        self.local_store = Node(local_store_path,
                                verify_exists_hook=verify_exists_hook)
        self.backup_stores_path = backup_stores_path

        backup_nodes = self.nodes[:]
        backup_nodes.remove(self.local_node)
        self.backup_stores = {}
        for backup_node in backup_nodes:
            backup_node_dir = backup_node.replace(':', '_')
            backup_path = os.path.join(backup_stores_path, backup_node_dir)
            if not os.path.exists(backup_path):
                os.makedirs(backup_path)
            self.backup_stores[backup_node] = Node(backup_path,
                                                   size=512 * 1024)

        self._register_rpc()
コード例 #7
0
ファイル: test_bloom_filter_mq.py プロジェクト: zouxc-zz/cola
    def testPutGet(self):
        num = str(12345)

        self.assertEqual(self.node.put(num), num)
        self.assertEqual(self.node.put(num), '')

        num2 = str(67890)
        nums = [num, num2]
        self.assertEqual(self.node.put(nums), [num2])

        self.node.shutdown()
        self.assertGreater(
            os.path.getsize(os.path.join(self.dir_, 'bloomfilter')), 0)

        bloom_filter_hook = FileBloomFilter(
            os.path.join(self.dir_, 'bloomfilter'), 5)
        self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)

        num3 = str(13579)
        nums = [num, num2, num3]
        self.assertEqual(self.node.put(nums), [num3])
コード例 #8
0
 def add_node(self, node, backup_store=None):
     self.nodes.append(node)
     self.hash_ring = HashRing(self.nodes)
     if backup_store is not None:
         self.backup_stores[node] = backup_store
     else:
         backup_stores_path = getattr(self, 'backup_stores_path')
         if backup_stores_path is not None:
             path = os.path.join(backup_stores_path, node.replace(':', '_'))
             if not os.path.exists(path):
                 os.makedirs(path)
             self.backup_stores[node] = Node(path, size=512 * 1024)
コード例 #9
0
ファイル: __init__.py プロジェクト: JanFan/cola
    def init_store(self, local_store_path, backup_stores_path, verify_exists_hook=None):
        self.local_store = Node(local_store_path, verify_exists_hook=verify_exists_hook)
        self.backup_stores_path = backup_stores_path

        backup_nodes = self.nodes[:]
        backup_nodes.remove(self.local_node)
        self.backup_stores = {}
        for backup_node in backup_nodes:
            backup_node_dir = backup_node.replace(":", "_")
            backup_path = os.path.join(backup_stores_path, backup_node_dir)
            if not os.path.exists(backup_path):
                os.makedirs(backup_path)
            self.backup_stores[backup_node] = Node(backup_path, size=512 * 1024)

        self._register_rpc()
コード例 #10
0
ファイル: test_bloom_filter_mq.py プロジェクト: 0pengl/cola
 def testPutGet(self):
     num = str(12345)
     
     self.assertEqual(self.node.put(num), num)
     self.assertEqual(self.node.put(num), '')
     
     num2 = str(67890)
     nums = [num, num2]
     self.assertEqual(self.node.put(nums), [num2])
     
     self.node.shutdown()
     self.assertGreater(os.path.getsize(os.path.join(self.dir_, 'bloomfilter')), 0)
     
     bloom_filter_hook = FileBloomFilter(
         os.path.join(self.dir_, 'bloomfilter'), 5
     )
     self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)
     
     num3 = str(13579)
     nums = [num, num2, num3]
     self.assertEqual(self.node.put(nums), [num3])
コード例 #11
0
ファイル: test_bloom_filter_mq.py プロジェクト: 0pengl/cola
class Test(unittest.TestCase):


    def setUp(self):
        self.dir_ = tempfile.mkdtemp()
        self.node_dir = os.path.join(self.dir_, 'node')
        os.mkdir(self.node_dir)
        bloom_filter_hook = FileBloomFilter(
            os.path.join(self.dir_, 'bloomfilter'), 10
        )
        self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)

    def tearDown(self):
        self.node.shutdown()
        shutil.rmtree(self.dir_)
          
    def testPutGet(self):
        num = str(12345)
        
        self.assertEqual(self.node.put(num), num)
        self.assertEqual(self.node.put(num), '')
        
        num2 = str(67890)
        nums = [num, num2]
        self.assertEqual(self.node.put(nums), [num2])
        
        self.node.shutdown()
        self.assertGreater(os.path.getsize(os.path.join(self.dir_, 'bloomfilter')), 0)
        
        bloom_filter_hook = FileBloomFilter(
            os.path.join(self.dir_, 'bloomfilter'), 5
        )
        self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)
        
        num3 = str(13579)
        nums = [num, num2, num3]
        self.assertEqual(self.node.put(nums), [num3])
コード例 #12
0
ファイル: __init__.py プロジェクト: ballacky13/cola
class MessageQueue(object):
    def __init__(self, nodes, local_node=None, rpc_server=None, 
                 local_store=None, backup_stores=None, copies=1):
        self.nodes = nodes
        self.local_node = local_node
        self.local_store = local_store
        self.rpc_server = rpc_server
        self.backup_stores = backup_stores
        self.hash_ring = HashRing(self.nodes)
        self.copies = max(min(len(self.nodes)-1, copies), 0)
        
        if rpc_server is not None and \
            self.local_store is not None and \
            self.backup_stores is not None:
            self._register_rpc()
            
    def _register_rpc(self):
        self.rpc_server.register_function(self.put_backup, 'put_backup')
        self.rpc_server.register_instance(self.local_store)
            
    def init_store(self, local_store_path, backup_stores_path, 
                   verify_exists_hook=None):
        self.local_store = Node(local_store_path, 
                                verify_exists_hook=verify_exists_hook)
        self.backup_stores_path = backup_stores_path
        
        backup_nodes = self.nodes[:]
        backup_nodes.remove(self.local_node)
        self.backup_stores = {}
        for backup_node in backup_nodes:
            backup_node_dir = backup_node.replace(':', '_')
            backup_path = os.path.join(backup_stores_path, backup_node_dir)
            if not os.path.exists(backup_path):
                os.makedirs(backup_path)
            self.backup_stores[backup_node] = Node(backup_path, 
                                                   size=512*1024)
            
        self._register_rpc()
        
    def _put(self, node, objs):
        if node == self.local_node:
            self.local_store.put(objs)
        else:
            client_call(node, 'put', objs)
                
    def _put_backup(self, node, src, objs):
        if node == self.local_node:
            self.put_backup(src, objs)
        else:
            client_call(node, 'put_backup', src, objs)
                
    def _get(self, node):
        if node == self.local_node:
            return self.local_store.get()
        else:
            return client_call(node, 'get')
        
    def put(self, obj_or_objs):
        def _check(obj):
            if not isinstance(obj, str):
                raise ValueError("MessageQueue can only put string objects.")
        if isinstance(obj_or_objs, (tuple, list)):
            for obj in obj_or_objs:
                _check(obj)
            objs = obj_or_objs
        else:
            _check(obj_or_objs)
            objs = [obj_or_objs]
        
        puts = {}
        bkup_puts = {}
        for obj in objs:
            it = self.hash_ring.iterate_nodes(obj)
            
            # put obj into an mq node.
            put_node = next(it)
            obs = puts.get(put_node, [])
            obs.append(obj)
            puts[put_node] = obs
            
            for _ in xrange(self.copies):
                bkup_node = next(it)
                if bkup_node is None: continue
                
                kv = bkup_puts.get(bkup_node, {})
                obs = kv.get(put_node, [])
                obs.append(obj)
                kv[put_node] = obs
                bkup_puts[bkup_node] = kv
        
        for k, v in puts.iteritems():
            self._put(k, v)
        for k, v in bkup_puts.iteritems():
            for src_node, obs in v.iteritems():
                self._put_backup(k, src_node, obs)
            
    def put_backup(self, src, obj_or_objs):
        backup_store = self.backup_stores[src]
        backup_store.put(obj_or_objs)
            
    def get(self):
        if self.local_node is not None:
            nodes = sorted(self.nodes, key=lambda k: k==self.local_node, reverse=True)
        else:
            nodes = self.nodes
        for n in nodes:
            obj = self._get(n)
            if obj is not None:
                return obj
            
    def remove_node(self, node):
        self.nodes.remove(node)
        self.hash_ring = HashRing(self.nodes)
        
        backup_store = self.backup_stores[node]
        obj = backup_store.get()
        while obj is not None:
            self.local_store.put(obj)
            obj = backup_store.get()
            
        backup_store.shutdown()
        del self.backup_stores[node]
        
    def add_node(self, node, backup_store=None):
        self.nodes.append(node)
        self.hash_ring = HashRing(self.nodes)
        if backup_store is not None:
            self.backup_stores[node] = backup_store
        else:
            backup_stores_path = getattr(self, 'backup_stores_path')
            if backup_stores_path is not None:
                path = os.path.join(backup_stores_path, node.replace(':', '_'))
                if not os.path.exists(path):
                    os.makedirs(path)
                self.backup_stores[node] = Node(path, 
                                                size=512*1024)
            
    def shutdown(self):
        self.local_store.shutdown()
        for backup_store in self.backup_stores.values():
            backup_store.shutdown()
            
    def __enter__(self):
        return self
    
    def __exit__(self):
        self.shutdown()
コード例 #13
0
class MessageQueue(object):
    def __init__(self,
                 nodes,
                 local_node=None,
                 rpc_server=None,
                 local_store=None,
                 backup_stores=None,
                 copies=1):
        self.nodes = nodes
        self.local_node = local_node
        self.local_store = local_store
        self.rpc_server = rpc_server
        self.backup_stores = backup_stores
        self.hash_ring = HashRing(self.nodes)
        self.copies = max(min(len(self.nodes) - 1, copies), 0)

        if rpc_server is not None and \
            self.local_store is not None and \
            self.backup_stores is not None:
            self._register_rpc()

    def _register_rpc(self):
        self.rpc_server.register_function(self.put_backup, 'put_backup')
        self.rpc_server.register_instance(self.local_store)

    def init_store(self,
                   local_store_path,
                   backup_stores_path,
                   verify_exists_hook=None):
        self.local_store = Node(local_store_path,
                                verify_exists_hook=verify_exists_hook)
        self.backup_stores_path = backup_stores_path

        backup_nodes = self.nodes[:]
        backup_nodes.remove(self.local_node)
        self.backup_stores = {}
        for backup_node in backup_nodes:
            backup_node_dir = backup_node.replace(':', '_')
            backup_path = os.path.join(backup_stores_path, backup_node_dir)
            if not os.path.exists(backup_path):
                os.makedirs(backup_path)
            self.backup_stores[backup_node] = Node(backup_path,
                                                   size=512 * 1024)

        self._register_rpc()

    def _put(self, node, objs):
        if node == self.local_node:
            self.local_store.put(objs)
        else:
            client_call(node, 'put', objs)

    def _put_backup(self, node, src, objs):
        if node == self.local_node:
            self.put_backup(src, objs)
        else:
            client_call(node, 'put_backup', src, objs)

    def _get(self, node):
        if node == self.local_node:
            return self.local_store.get()
        else:
            return client_call(node, 'get')

    def put(self, obj_or_objs):
        def _check(obj):
            if not isinstance(obj, str):
                raise ValueError("MessageQueue can only put string objects.")

        if isinstance(obj_or_objs, (tuple, list)):
            for obj in obj_or_objs:
                _check(obj)
            objs = obj_or_objs
        else:
            _check(obj_or_objs)
            objs = [obj_or_objs]

        puts = {}
        bkup_puts = {}
        for obj in objs:
            it = self.hash_ring.iterate_nodes(obj)

            # put obj into an mq node.
            put_node = next(it)
            obs = puts.get(put_node, [])
            obs.append(obj)
            puts[put_node] = obs

            for _ in xrange(self.copies):
                bkup_node = next(it)
                if bkup_node is None: continue

                kv = bkup_puts.get(bkup_node, {})
                obs = kv.get(put_node, [])
                obs.append(obj)
                kv[put_node] = obs
                bkup_puts[bkup_node] = kv

        for k, v in puts.iteritems():
            self._put(k, v)
        for k, v in bkup_puts.iteritems():
            for src_node, obs in v.iteritems():
                self._put_backup(k, src_node, obs)

    def put_backup(self, src, obj_or_objs):
        backup_store = self.backup_stores[src]
        backup_store.put(obj_or_objs)

    def get(self):
        if self.local_node is not None:
            nodes = sorted(self.nodes,
                           key=lambda k: k == self.local_node,
                           reverse=True)
        else:
            nodes = self.nodes
        for n in nodes:
            obj = self._get(n)
            if obj is not None:
                return obj

    def remove_node(self, node):
        self.nodes.remove(node)
        self.hash_ring = HashRing(self.nodes)

        backup_store = self.backup_stores[node]
        obj = backup_store.get()
        while obj is not None:
            self.local_store.put(obj)
            obj = backup_store.get()

        backup_store.shutdown()
        del self.backup_stores[node]

    def add_node(self, node, backup_store=None):
        self.nodes.append(node)
        self.hash_ring = HashRing(self.nodes)
        if backup_store is not None:
            self.backup_stores[node] = backup_store
        else:
            backup_stores_path = getattr(self, 'backup_stores_path')
            if backup_stores_path is not None:
                path = os.path.join(backup_stores_path, node.replace(':', '_'))
                if not os.path.exists(path):
                    os.makedirs(path)
                self.backup_stores[node] = Node(path, size=512 * 1024)

    def shutdown(self):
        self.local_store.shutdown()
        for backup_store in self.backup_stores.values():
            backup_store.shutdown()

    def __enter__(self):
        return self

    def __exit__(self):
        self.shutdown()
コード例 #14
0
ファイル: test_mq_node.py プロジェクト: zouxc-zz/cola
 def setUp(self):
     self.dir_ = tempfile.mkdtemp()
     self.node = Node(self.dir_)
コード例 #15
0
ファイル: test_mq_node.py プロジェクト: zouxc-zz/cola
class Test(unittest.TestCase):
    def setUp(self):
        self.dir_ = tempfile.mkdtemp()
        self.node = Node(self.dir_)

    def tearDown(self):
        self.node.shutdown()
        shutil.rmtree(self.dir_)

    def testLockExists(self):
        self.assertTrue(os.path.exists(os.path.join(self.dir_, 'lock')))

    def testPutGet(self):
        get_num = lambda: random.randint(10000, 20000)

        num1 = get_num()
        self.assertEqual(self.node.put(str(num1)), str(num1))
        num2 = get_num()
        self.assertEqual(self.node.put(str(num2)), str(num2))

        self.assertEqual(self.node.get(), str(num1))
        self.assertEqual(self.node.get(), str(num2))

    def testBatchPutGet(self):
        self.node.shutdown()

        size = 50
        batch1 = ['1' * 20, '2' * 20]
        batch2 = ['3' * 20, '4' * 20]

        self.node = Node(self.dir_, size)

        self.assertEqual(self.node.put(batch1), batch1)
        self.assertEqual(self.node.put(batch2), batch2)

        self.assertEqual(len(self.node.map_files), 2)

        gets = sorted([self.node.get() for _ in range(4)])
        res = batch1
        res.extend(batch2)
        self.assertEqual(gets, res)

        self.node.put('5' * 20)
        self.assertEqual(self.node.get(), '5' * 20)

        self.node.put('6' * 20)

        self.node.merge()
        self.assertEqual(len(self.node.map_files), 1)

        self.assertEqual(self.node.get(), '6' * 20)
        self.assertEqual(self.node.get(), None)

        self.assertRaises(NodeNoSpaceForPut, lambda: self.node.put('7' * 100))
コード例 #16
0
ファイル: test_mq_node.py プロジェクト: 0pengl/cola
class Test(unittest.TestCase):


    def setUp(self):
        self.dir_ = tempfile.mkdtemp()
        self.node = Node(self.dir_)

    def tearDown(self):
        self.node.shutdown()
        shutil.rmtree(self.dir_)

    def testLockExists(self):
        self.assertTrue(os.path.exists(os.path.join(self.dir_, 'lock')))
          
    def testPutGet(self):
        get_num = lambda: random.randint(10000, 20000)
          
        num1 = get_num()
        self.assertEqual(self.node.put(str(num1)), str(num1))
        num2 = get_num()
        self.assertEqual(self.node.put(str(num2)), str(num2))
          
        self.assertEqual(self.node.get(), str(num1))
        self.assertEqual(self.node.get(), str(num2))
        
    def testBatchPutGet(self):
        self.node.shutdown()
        
        size = 50
        batch1 = ['1' * 20, '2' * 20]
        batch2 = ['3' * 20, '4' * 20]
        
        self.node = Node(self.dir_, size)
        
        self.assertEqual(self.node.put(batch1), batch1)
        self.assertEqual(self.node.put(batch2), batch2)
        
        self.assertEqual(len(self.node.map_files), 2)
        
        gets = sorted([self.node.get() for _ in range(4)])
        res = batch1
        res.extend(batch2)
        self.assertEqual(gets, res)
        
        self.node.put('5' * 20)
        self.assertEqual(self.node.get(), '5' * 20)
        
        self.node.put('6' * 20)
        
        self.node.merge()
        self.assertEqual(len(self.node.map_files), 1)
        
        self.assertEqual(self.node.get(), '6' * 20)
        self.assertEqual(self.node.get(), None)
        
        self.assertRaises(NodeNoSpaceForPut, lambda: self.node.put('7' * 100))
コード例 #17
0
ファイル: test_mq_node.py プロジェクト: 0pengl/cola
 def setUp(self):
     self.dir_ = tempfile.mkdtemp()
     self.node = Node(self.dir_)