def setUp(self):
self.dir_ = tempfile.mkdtemp()
self.node_dir = os.path.join(self.dir_, 'node')
os.mkdir(self.node_dir)
bloom_filter_hook = FileBloomFilter(
os.path.join(self.dir_, 'bloomfilter'), 10)
self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)
def testBatchPutGet(self):
self.node.shutdown()
size = 50
batch1 = ['1' * 20, '2' * 20]
batch2 = ['3' * 20, '4' * 20]
self.node = Node(self.dir_, size)
self.assertEqual(self.node.put(batch1), batch1)
self.assertEqual(self.node.put(batch2), batch2)
self.assertEqual(len(self.node.map_files), 2)
gets = sorted([self.node.get() for _ in range(4)])
res = batch1
res.extend(batch2)
self.assertEqual(gets, res)
self.node.put('5' * 20)
self.assertEqual(self.node.get(), '5' * 20)
self.node.put('6' * 20)
self.node.merge()
self.assertEqual(len(self.node.map_files), 1)
self.assertEqual(self.node.get(), '6' * 20)
self.assertEqual(self.node.get(), None)
self.assertRaises(NodeNoSpaceForPut, lambda: self.node.put('7' * 100))
def testBatchPutGet(self):
self.node.shutdown()
size = 50
batch1 = ['1' * 20, '2' * 20]
batch2 = ['3' * 20, '4' * 20]
self.node = Node(self.dir_, size)
self.assertEqual(self.node.put(batch1), batch1)
self.assertEqual(self.node.put(batch2), batch2)
self.assertEqual(len(self.node.map_files), 2)
gets = sorted([self.node.get() for _ in range(4)])
res = batch1
res.extend(batch2)
self.assertEqual(gets, res)
self.node.put('5' * 20)
self.assertEqual(self.node.get(), '5' * 20)
self.node.put('6' * 20)
self.node.merge()
self.assertEqual(len(self.node.map_files), 1)
self.assertEqual(self.node.get(), '6' * 20)
self.assertEqual(self.node.get(), None)
self.assertRaises(NodeNoSpaceForPut, lambda: self.node.put('7' * 100))
def setUp(self):
self.dir_ = tempfile.mkdtemp()
self.node_dir = os.path.join(self.dir_, 'node')
os.mkdir(self.node_dir)
bloom_filter_hook = FileBloomFilter(
os.path.join(self.dir_, 'bloomfilter'), 10
)
self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)
class Test(unittest.TestCase):
def setUp(self):
self.dir_ = tempfile.mkdtemp()
self.node_dir = os.path.join(self.dir_, 'node')
os.mkdir(self.node_dir)
bloom_filter_hook = FileBloomFilter(
os.path.join(self.dir_, 'bloomfilter'), 10)
self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)
def tearDown(self):
self.node.shutdown()
shutil.rmtree(self.dir_)
def testPutGet(self):
num = str(12345)
self.assertEqual(self.node.put(num), num)
self.assertEqual(self.node.put(num), '')
num2 = str(67890)
nums = [num, num2]
self.assertEqual(self.node.put(nums), [num2])
self.node.shutdown()
self.assertGreater(
os.path.getsize(os.path.join(self.dir_, 'bloomfilter')), 0)
bloom_filter_hook = FileBloomFilter(
os.path.join(self.dir_, 'bloomfilter'), 5)
self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)
num3 = str(13579)
nums = [num, num2, num3]
self.assertEqual(self.node.put(nums), [num3])
def init_store(self,
local_store_path,
backup_stores_path,
verify_exists_hook=None):
self.local_store = Node(local_store_path,
verify_exists_hook=verify_exists_hook)
self.backup_stores_path = backup_stores_path
backup_nodes = self.nodes[:]
backup_nodes.remove(self.local_node)
self.backup_stores = {}
for backup_node in backup_nodes:
backup_node_dir = backup_node.replace(':', '_')
backup_path = os.path.join(backup_stores_path, backup_node_dir)
if not os.path.exists(backup_path):
os.makedirs(backup_path)
self.backup_stores[backup_node] = Node(backup_path,
size=512 * 1024)
self._register_rpc()
def testPutGet(self):
num = str(12345)
self.assertEqual(self.node.put(num), num)
self.assertEqual(self.node.put(num), '')
num2 = str(67890)
nums = [num, num2]
self.assertEqual(self.node.put(nums), [num2])
self.node.shutdown()
self.assertGreater(
os.path.getsize(os.path.join(self.dir_, 'bloomfilter')), 0)
bloom_filter_hook = FileBloomFilter(
os.path.join(self.dir_, 'bloomfilter'), 5)
self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)
num3 = str(13579)
nums = [num, num2, num3]
self.assertEqual(self.node.put(nums), [num3])
def add_node(self, node, backup_store=None):
self.nodes.append(node)
self.hash_ring = HashRing(self.nodes)
if backup_store is not None:
self.backup_stores[node] = backup_store
else:
backup_stores_path = getattr(self, 'backup_stores_path')
if backup_stores_path is not None:
path = os.path.join(backup_stores_path, node.replace(':', '_'))
if not os.path.exists(path):
os.makedirs(path)
self.backup_stores[node] = Node(path, size=512 * 1024)
def init_store(self, local_store_path, backup_stores_path, verify_exists_hook=None):
self.local_store = Node(local_store_path, verify_exists_hook=verify_exists_hook)
self.backup_stores_path = backup_stores_path
backup_nodes = self.nodes[:]
backup_nodes.remove(self.local_node)
self.backup_stores = {}
for backup_node in backup_nodes:
backup_node_dir = backup_node.replace(":", "_")
backup_path = os.path.join(backup_stores_path, backup_node_dir)
if not os.path.exists(backup_path):
os.makedirs(backup_path)
self.backup_stores[backup_node] = Node(backup_path, size=512 * 1024)
self._register_rpc()
def testPutGet(self):
num = str(12345)
self.assertEqual(self.node.put(num), num)
self.assertEqual(self.node.put(num), '')
num2 = str(67890)
nums = [num, num2]
self.assertEqual(self.node.put(nums), [num2])
self.node.shutdown()
self.assertGreater(os.path.getsize(os.path.join(self.dir_, 'bloomfilter')), 0)
bloom_filter_hook = FileBloomFilter(
os.path.join(self.dir_, 'bloomfilter'), 5
)
self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)
num3 = str(13579)
nums = [num, num2, num3]
self.assertEqual(self.node.put(nums), [num3])
class Test(unittest.TestCase):
def setUp(self):
self.dir_ = tempfile.mkdtemp()
self.node_dir = os.path.join(self.dir_, 'node')
os.mkdir(self.node_dir)
bloom_filter_hook = FileBloomFilter(
os.path.join(self.dir_, 'bloomfilter'), 10
)
self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)
def tearDown(self):
self.node.shutdown()
shutil.rmtree(self.dir_)
def testPutGet(self):
num = str(12345)
self.assertEqual(self.node.put(num), num)
self.assertEqual(self.node.put(num), '')
num2 = str(67890)
nums = [num, num2]
self.assertEqual(self.node.put(nums), [num2])
self.node.shutdown()
self.assertGreater(os.path.getsize(os.path.join(self.dir_, 'bloomfilter')), 0)
bloom_filter_hook = FileBloomFilter(
os.path.join(self.dir_, 'bloomfilter'), 5
)
self.node = Node(self.node_dir, verify_exists_hook=bloom_filter_hook)
num3 = str(13579)
nums = [num, num2, num3]
self.assertEqual(self.node.put(nums), [num3])
class MessageQueue(object):
def __init__(self, nodes, local_node=None, rpc_server=None,
local_store=None, backup_stores=None, copies=1):
self.nodes = nodes
self.local_node = local_node
self.local_store = local_store
self.rpc_server = rpc_server
self.backup_stores = backup_stores
self.hash_ring = HashRing(self.nodes)
self.copies = max(min(len(self.nodes)-1, copies), 0)
if rpc_server is not None and \
self.local_store is not None and \
self.backup_stores is not None:
self._register_rpc()
def _register_rpc(self):
self.rpc_server.register_function(self.put_backup, 'put_backup')
self.rpc_server.register_instance(self.local_store)
def init_store(self, local_store_path, backup_stores_path,
verify_exists_hook=None):
self.local_store = Node(local_store_path,
verify_exists_hook=verify_exists_hook)
self.backup_stores_path = backup_stores_path
backup_nodes = self.nodes[:]
backup_nodes.remove(self.local_node)
self.backup_stores = {}
for backup_node in backup_nodes:
backup_node_dir = backup_node.replace(':', '_')
backup_path = os.path.join(backup_stores_path, backup_node_dir)
if not os.path.exists(backup_path):
os.makedirs(backup_path)
self.backup_stores[backup_node] = Node(backup_path,
size=512*1024)
self._register_rpc()
def _put(self, node, objs):
if node == self.local_node:
self.local_store.put(objs)
else:
client_call(node, 'put', objs)
def _put_backup(self, node, src, objs):
if node == self.local_node:
self.put_backup(src, objs)
else:
client_call(node, 'put_backup', src, objs)
def _get(self, node):
if node == self.local_node:
return self.local_store.get()
else:
return client_call(node, 'get')
def put(self, obj_or_objs):
def _check(obj):
if not isinstance(obj, str):
raise ValueError("MessageQueue can only put string objects.")
if isinstance(obj_or_objs, (tuple, list)):
for obj in obj_or_objs:
_check(obj)
objs = obj_or_objs
else:
_check(obj_or_objs)
objs = [obj_or_objs]
puts = {}
bkup_puts = {}
for obj in objs:
it = self.hash_ring.iterate_nodes(obj)
# put obj into an mq node.
put_node = next(it)
obs = puts.get(put_node, [])
obs.append(obj)
puts[put_node] = obs
for _ in xrange(self.copies):
bkup_node = next(it)
if bkup_node is None: continue
kv = bkup_puts.get(bkup_node, {})
obs = kv.get(put_node, [])
obs.append(obj)
kv[put_node] = obs
bkup_puts[bkup_node] = kv
for k, v in puts.iteritems():
self._put(k, v)
for k, v in bkup_puts.iteritems():
for src_node, obs in v.iteritems():
self._put_backup(k, src_node, obs)
def put_backup(self, src, obj_or_objs):
backup_store = self.backup_stores[src]
backup_store.put(obj_or_objs)
def get(self):
if self.local_node is not None:
nodes = sorted(self.nodes, key=lambda k: k==self.local_node, reverse=True)
else:
nodes = self.nodes
for n in nodes:
obj = self._get(n)
if obj is not None:
return obj
def remove_node(self, node):
self.nodes.remove(node)
self.hash_ring = HashRing(self.nodes)
backup_store = self.backup_stores[node]
obj = backup_store.get()
while obj is not None:
self.local_store.put(obj)
obj = backup_store.get()
backup_store.shutdown()
del self.backup_stores[node]
def add_node(self, node, backup_store=None):
self.nodes.append(node)
self.hash_ring = HashRing(self.nodes)
if backup_store is not None:
self.backup_stores[node] = backup_store
else:
backup_stores_path = getattr(self, 'backup_stores_path')
if backup_stores_path is not None:
path = os.path.join(backup_stores_path, node.replace(':', '_'))
if not os.path.exists(path):
os.makedirs(path)
self.backup_stores[node] = Node(path,
size=512*1024)
def shutdown(self):
self.local_store.shutdown()
for backup_store in self.backup_stores.values():
backup_store.shutdown()
def __enter__(self):
return self
def __exit__(self):
self.shutdown()
class MessageQueue(object):
def __init__(self,
nodes,
local_node=None,
rpc_server=None,
local_store=None,
backup_stores=None,
copies=1):
self.nodes = nodes
self.local_node = local_node
self.local_store = local_store
self.rpc_server = rpc_server
self.backup_stores = backup_stores
self.hash_ring = HashRing(self.nodes)
self.copies = max(min(len(self.nodes) - 1, copies), 0)
if rpc_server is not None and \
self.local_store is not None and \
self.backup_stores is not None:
self._register_rpc()
def _register_rpc(self):
self.rpc_server.register_function(self.put_backup, 'put_backup')
self.rpc_server.register_instance(self.local_store)
def init_store(self,
local_store_path,
backup_stores_path,
verify_exists_hook=None):
self.local_store = Node(local_store_path,
verify_exists_hook=verify_exists_hook)
self.backup_stores_path = backup_stores_path
backup_nodes = self.nodes[:]
backup_nodes.remove(self.local_node)
self.backup_stores = {}
for backup_node in backup_nodes:
backup_node_dir = backup_node.replace(':', '_')
backup_path = os.path.join(backup_stores_path, backup_node_dir)
if not os.path.exists(backup_path):
os.makedirs(backup_path)
self.backup_stores[backup_node] = Node(backup_path,
size=512 * 1024)
self._register_rpc()
def _put(self, node, objs):
if node == self.local_node:
self.local_store.put(objs)
else:
client_call(node, 'put', objs)
def _put_backup(self, node, src, objs):
if node == self.local_node:
self.put_backup(src, objs)
else:
client_call(node, 'put_backup', src, objs)
def _get(self, node):
if node == self.local_node:
return self.local_store.get()
else:
return client_call(node, 'get')
def put(self, obj_or_objs):
def _check(obj):
if not isinstance(obj, str):
raise ValueError("MessageQueue can only put string objects.")
if isinstance(obj_or_objs, (tuple, list)):
for obj in obj_or_objs:
_check(obj)
objs = obj_or_objs
else:
_check(obj_or_objs)
objs = [obj_or_objs]
puts = {}
bkup_puts = {}
for obj in objs:
it = self.hash_ring.iterate_nodes(obj)
# put obj into an mq node.
put_node = next(it)
obs = puts.get(put_node, [])
obs.append(obj)
puts[put_node] = obs
for _ in xrange(self.copies):
bkup_node = next(it)
if bkup_node is None: continue
kv = bkup_puts.get(bkup_node, {})
obs = kv.get(put_node, [])
obs.append(obj)
kv[put_node] = obs
bkup_puts[bkup_node] = kv
for k, v in puts.iteritems():
self._put(k, v)
for k, v in bkup_puts.iteritems():
for src_node, obs in v.iteritems():
self._put_backup(k, src_node, obs)
def put_backup(self, src, obj_or_objs):
backup_store = self.backup_stores[src]
backup_store.put(obj_or_objs)
def get(self):
if self.local_node is not None:
nodes = sorted(self.nodes,
key=lambda k: k == self.local_node,
reverse=True)
else:
nodes = self.nodes
for n in nodes:
obj = self._get(n)
if obj is not None:
return obj
def remove_node(self, node):
self.nodes.remove(node)
self.hash_ring = HashRing(self.nodes)
backup_store = self.backup_stores[node]
obj = backup_store.get()
while obj is not None:
self.local_store.put(obj)
obj = backup_store.get()
backup_store.shutdown()
del self.backup_stores[node]
def add_node(self, node, backup_store=None):
self.nodes.append(node)
self.hash_ring = HashRing(self.nodes)
if backup_store is not None:
self.backup_stores[node] = backup_store
else:
backup_stores_path = getattr(self, 'backup_stores_path')
if backup_stores_path is not None:
path = os.path.join(backup_stores_path, node.replace(':', '_'))
if not os.path.exists(path):
os.makedirs(path)
self.backup_stores[node] = Node(path, size=512 * 1024)
def shutdown(self):
self.local_store.shutdown()
for backup_store in self.backup_stores.values():
backup_store.shutdown()
def __enter__(self):
return self
def __exit__(self):
self.shutdown()
def setUp(self):
self.dir_ = tempfile.mkdtemp()
self.node = Node(self.dir_)
class Test(unittest.TestCase):
def setUp(self):
self.dir_ = tempfile.mkdtemp()
self.node = Node(self.dir_)
def tearDown(self):
self.node.shutdown()
shutil.rmtree(self.dir_)
def testLockExists(self):
self.assertTrue(os.path.exists(os.path.join(self.dir_, 'lock')))
def testPutGet(self):
get_num = lambda: random.randint(10000, 20000)
num1 = get_num()
self.assertEqual(self.node.put(str(num1)), str(num1))
num2 = get_num()
self.assertEqual(self.node.put(str(num2)), str(num2))
self.assertEqual(self.node.get(), str(num1))
self.assertEqual(self.node.get(), str(num2))
def testBatchPutGet(self):
self.node.shutdown()
size = 50
batch1 = ['1' * 20, '2' * 20]
batch2 = ['3' * 20, '4' * 20]
self.node = Node(self.dir_, size)
self.assertEqual(self.node.put(batch1), batch1)
self.assertEqual(self.node.put(batch2), batch2)
self.assertEqual(len(self.node.map_files), 2)
gets = sorted([self.node.get() for _ in range(4)])
res = batch1
res.extend(batch2)
self.assertEqual(gets, res)
self.node.put('5' * 20)
self.assertEqual(self.node.get(), '5' * 20)
self.node.put('6' * 20)
self.node.merge()
self.assertEqual(len(self.node.map_files), 1)
self.assertEqual(self.node.get(), '6' * 20)
self.assertEqual(self.node.get(), None)
self.assertRaises(NodeNoSpaceForPut, lambda: self.node.put('7' * 100))
class Test(unittest.TestCase):
def setUp(self):
self.dir_ = tempfile.mkdtemp()
self.node = Node(self.dir_)
def tearDown(self):
self.node.shutdown()
shutil.rmtree(self.dir_)
def testLockExists(self):
self.assertTrue(os.path.exists(os.path.join(self.dir_, 'lock')))
def testPutGet(self):
get_num = lambda: random.randint(10000, 20000)
num1 = get_num()
self.assertEqual(self.node.put(str(num1)), str(num1))
num2 = get_num()
self.assertEqual(self.node.put(str(num2)), str(num2))
self.assertEqual(self.node.get(), str(num1))
self.assertEqual(self.node.get(), str(num2))
def testBatchPutGet(self):
self.node.shutdown()
size = 50
batch1 = ['1' * 20, '2' * 20]
batch2 = ['3' * 20, '4' * 20]
self.node = Node(self.dir_, size)
self.assertEqual(self.node.put(batch1), batch1)
self.assertEqual(self.node.put(batch2), batch2)
self.assertEqual(len(self.node.map_files), 2)
gets = sorted([self.node.get() for _ in range(4)])
res = batch1
res.extend(batch2)
self.assertEqual(gets, res)
self.node.put('5' * 20)
self.assertEqual(self.node.get(), '5' * 20)
self.node.put('6' * 20)
self.node.merge()
self.assertEqual(len(self.node.map_files), 1)
self.assertEqual(self.node.get(), '6' * 20)
self.assertEqual(self.node.get(), None)
self.assertRaises(NodeNoSpaceForPut, lambda: self.node.put('7' * 100))
def setUp(self):
self.dir_ = tempfile.mkdtemp()
self.node = Node(self.dir_)