def test_remove(self): executor = distributed.Executor('127.0.0.1:8786') yp = df.YellowPages(executor) yp.allocate(['block{}'.format(i) for i in range(10)], 1024) yp.insert('x', 4) yp.find('x') # Should not raise an exception self.assertEqual((1024, 4, 4, 0), yp.find('x', ip_only=False)[2:]) yp.remove('x') with self.assertRaises(LookupError): yp.find('x') yp.insert('x', 4) yp.insert('y', 8) yp.find('x') # Should not raise an exception self.assertEqual((1024, 12, 4, 4), yp.find('x', ip_only=False)[2:]) yp.find('y') # Should not raise an exception self.assertEqual((1024, 12, 8, 8), yp.find('y', ip_only=False)[2:]) yp.remove('x') self.assertEqual((1024, 8, 8, 8), yp.find('y', ip_only=False)[2:]) with self.assertRaises(LookupError): yp.find('x') yp.remove('y') with self.assertRaises(LookupError): yp.find('y') yp.clear()
def _get_distributed_executor(scheduler, workers): try: import distributed except ImportError: return None try: executor = DistributedExecutor(distributed.Executor(scheduler)) return executor except IOError: return None
def _get_distributed_executor(scheduler, workers): try: import distributed except ImportError: return None class DistributedExecutor(object): def __init__(self, executor): """ :type executor: distributed.Executor :return: """ self._executor = executor def submit(self, func, *args, **kwargs): return self._executor.submit(func, *args, **kwargs) def map(self, func, iterable): return self._executor.map(func, iterable) @staticmethod def as_completed(futures): return distributed.as_completed(futures) @classmethod def next_completed(cls, futures, default): results = list(futures) if not results: return default, results result = next(cls.as_completed(results), default) results.remove(result) return result, results @staticmethod def result(value): return value.result() @staticmethod def release(value): value.release() try: executor = DistributedExecutor(distributed.Executor(scheduler)) return executor except IOError: return None
def test_allocate(self): executor = distributed.Executor('127.0.0.1:8786') yp = df.YellowPages(executor) yp.allocate(['block{}'.format(i) for i in range(10)], 1024) yp.allocate(['_block{}'.format(i) for i in range(5)], 2048) blocks = [block[1] for block in yp.blocks()] self.assertTrue(len(blocks) == 15) for i in range(10): self.assertTrue('block{}'.format(i) in blocks) for i in range(5): self.assertTrue('_block{}'.format(i) in blocks) blocks = [block[1:] for block in yp.blocks(ip_only=False)] self.assertTrue(len(blocks) == 15) for i in range(10): self.assertTrue(('block{}'.format(i), 1024, 0) in blocks) for i in range(5): self.assertTrue(('_block{}'.format(i), 2048, 0) in blocks) yp.clear() self.assertTrue(not yp.blocks())
def get_executor(self, **kwargs): """ Return an associated executor """ return distributed.Executor(self.cluster.scheduler_address, **kwargs)