def test_remove(self):
executor = distributed.Executor('127.0.0.1:8786')
yp = df.YellowPages(executor)
yp.allocate(['block{}'.format(i) for i in range(10)], 1024)
yp.insert('x', 4)
yp.find('x') # Should not raise an exception
self.assertEqual((1024, 4, 4, 0), yp.find('x', ip_only=False)[2:])
yp.remove('x')
with self.assertRaises(LookupError):
yp.find('x')
yp.insert('x', 4)
yp.insert('y', 8)
yp.find('x') # Should not raise an exception
self.assertEqual((1024, 12, 4, 4), yp.find('x', ip_only=False)[2:])
yp.find('y') # Should not raise an exception
self.assertEqual((1024, 12, 8, 8), yp.find('y', ip_only=False)[2:])
yp.remove('x')
self.assertEqual((1024, 8, 8, 8), yp.find('y', ip_only=False)[2:])
with self.assertRaises(LookupError):
yp.find('x')
yp.remove('y')
with self.assertRaises(LookupError):
yp.find('y')
yp.clear()
def _get_distributed_executor(scheduler, workers):
try:
import distributed
except ImportError:
return None
try:
executor = DistributedExecutor(distributed.Executor(scheduler))
return executor
except IOError:
return None
def _get_distributed_executor(scheduler, workers):
try:
import distributed
except ImportError:
return None
class DistributedExecutor(object):
def __init__(self, executor):
"""
:type executor: distributed.Executor
:return:
"""
self._executor = executor
def submit(self, func, *args, **kwargs):
return self._executor.submit(func, *args, **kwargs)
def map(self, func, iterable):
return self._executor.map(func, iterable)
@staticmethod
def as_completed(futures):
return distributed.as_completed(futures)
@classmethod
def next_completed(cls, futures, default):
results = list(futures)
if not results:
return default, results
result = next(cls.as_completed(results), default)
results.remove(result)
return result, results
@staticmethod
def result(value):
return value.result()
@staticmethod
def release(value):
value.release()
try:
executor = DistributedExecutor(distributed.Executor(scheduler))
return executor
except IOError:
return None
def test_allocate(self):
executor = distributed.Executor('127.0.0.1:8786')
yp = df.YellowPages(executor)
yp.allocate(['block{}'.format(i) for i in range(10)], 1024)
yp.allocate(['_block{}'.format(i) for i in range(5)], 2048)
blocks = [block[1] for block in yp.blocks()]
self.assertTrue(len(blocks) == 15)
for i in range(10):
self.assertTrue('block{}'.format(i) in blocks)
for i in range(5):
self.assertTrue('_block{}'.format(i) in blocks)
blocks = [block[1:] for block in yp.blocks(ip_only=False)]
self.assertTrue(len(blocks) == 15)
for i in range(10):
self.assertTrue(('block{}'.format(i), 1024, 0) in blocks)
for i in range(5):
self.assertTrue(('_block{}'.format(i), 2048, 0) in blocks)
yp.clear()
self.assertTrue(not yp.blocks())
def get_executor(self, **kwargs):
""" Return an associated executor """
return distributed.Executor(self.cluster.scheduler_address, **kwargs)