def test_redis_entry_point(self):
conf = {
'board': 'redis',
}
p = impl_memory.MemoryBackend()
with contextlib.closing(backends.fetch('test', conf,
persistence=p)) as be:
self.assertIsInstance(be, impl_redis.RedisJobBoard)
def test_zk_entry_point(self):
p = impl_memory.MemoryBackend()
conf = {
'board': 'zookeeper',
}
with contextlib.closing(backends.fetch('test', conf,
persistence=p)) as be:
self.assertIsInstance(be, impl_zookeeper.ZookeeperJobBoard)
def setUp(self):
super(TestClaimListener, self).setUp()
self.client = fake_client.FakeClient()
self.addCleanup(self.client.stop)
self.board = jobs.fetch('test',
'zookeeper',
client=self.client,
persistence=impl_memory.MemoryBackend())
self.addCleanup(self.board.close)
self.board.connect()
def test_deregister(self):
"""Verify that register and deregister don't blow up"""
with contextlib.closing(impl_memory.MemoryBackend()) as be:
flow = lf.Flow("test")
flow.add(SleepyTask("test-1", sleep_for=0.1))
(lb, fd) = persistence_utils.temporary_flow_detail(be)
e = self._make_engine(flow, fd, be)
l = timing.DurationListener(e)
l.register()
l.deregister()
def create_board(self, persistence=None):
if persistence is None:
persistence = impl_memory.MemoryBackend()
client = fake_client.FakeClient()
board = impl_zookeeper.ZookeeperJobBoard('test-board', {},
client=client,
persistence=persistence)
self.addCleanup(board.close)
self.addCleanup(self.close_client, client)
return (client, board)
def make_components(self):
client = fake_client.FakeClient()
persistence = impl_memory.MemoryBackend()
board = impl_zookeeper.ZookeeperJobBoard('testing', {},
client=client,
persistence=persistence)
conductor_kwargs = self.conductor_kwargs.copy()
conductor_kwargs['persistence'] = persistence
conductor = backends.fetch(self.kind, 'testing', board,
**conductor_kwargs)
return ComponentBundle(board, client, persistence, conductor)
def test_flow_duration(self):
with contextlib.closing(impl_memory.MemoryBackend()) as be:
flow = lf.Flow("test")
flow.add(SleepyTask("test-1", sleep_for=0.1))
(lb, fd) = persistence_utils.temporary_flow_detail(be)
e = self._make_engine(flow, fd, be)
with timing.DurationListener(e):
e.run()
self.assertIsNotNone(fd)
self.assertIsNotNone(fd.meta)
self.assertIn('duration', fd.meta)
self.assertGreaterEqual(0.1, fd.meta['duration'])
def test_zk_entry_point_existing_client(self):
existing_client = fake_client.FakeClient()
conf = {
'board': 'zookeeper',
}
kwargs = {
'client': existing_client,
'persistence': impl_memory.MemoryBackend(),
}
with contextlib.closing(backends.fetch('test', conf, **kwargs)) as be:
self.assertIsInstance(be, impl_zookeeper.ZookeeperJobBoard)
self.assertIs(existing_client, be._client)
def test_bad_factory(self):
persistence = impl_memory.MemoryBackend()
client = fake_client.FakeClient()
board = impl_zookeeper.ZookeeperJobBoard('testing', {},
client=client,
persistence=persistence)
self.assertRaises(ValueError,
backends.fetch,
'nonblocking',
'testing',
board,
persistence=persistence,
executor_factory='testing')
def test_storage_progress(self):
with contextlib.closing(impl_memory.MemoryBackend({})) as be:
flo = lf.Flow("test")
flo.add(ProgressTask("test", 3))
b, fd = p_utils.temporary_flow_detail(be)
e = self._make_engine(flo, flow_detail=fd, backend=be)
e.run()
end_progress = e.storage.get_task_progress("test")
self.assertEqual(1.0, end_progress)
task_uuid = e.storage.get_atom_uuid("test")
td = fd.find(task_uuid)
self.assertEqual(1.0, td.meta['progress'])
self.assertFalse(td.meta['progress_details'])
def make_components(self, listener_factories):
client = fake_client.FakeClient()
persistence = impl_memory.MemoryBackend()
board = impl_zookeeper.ZookeeperJobBoard('testing', {},
client=client,
persistence=persistence)
conductor_kwargs = {
'wait_timeout': 0.01,
'listener_factories': listener_factories,
'persistence': persistence,
}
conductor = backends.fetch('blocking', 'testing', board,
**conductor_kwargs)
return ComponentBundle(board, client, persistence, conductor)
def create_board(self, persistence=None):
if persistence is None:
persistence = impl_memory.MemoryBackend()
namespace = uuidutils.generate_uuid()
client = ru.RedisClient()
config = {
'namespace': six.b("zag-%s" % namespace),
}
kwargs = {
'client': client,
'persistence': persistence,
}
board = impl_redis.RedisJobBoard('test-board', config, **kwargs)
self.addCleanup(board.close)
self.addCleanup(self.close_client, client)
return (client, board)
def test_record_ending_exception(self, mocked_warning):
with contextlib.closing(impl_memory.MemoryBackend()) as be:
flow = lf.Flow("test")
flow.add(test_utils.TaskNoRequiresNoReturns("test-1"))
(lb, fd) = persistence_utils.temporary_flow_detail(be)
e = self._make_engine(flow, fd, be)
duration_listener = timing.DurationListener(e)
with mock.patch.object(duration_listener._engine.storage,
'update_atom_metadata') as mocked_uam:
mocked_uam.side_effect = exc.StorageFailure('Woot!')
with duration_listener:
e.run()
mocked_warning.assert_called_once_with(mock.ANY,
mock.ANY,
'task',
'test-1',
exc_info=True)
def test_dual_storage_progress(self):
fired_events = []
def notify_me(event_type, details):
fired_events.append(details.pop('progress'))
with contextlib.closing(impl_memory.MemoryBackend({})) as be:
t = ProgressTask("test", 5)
t.notifier.register(task.EVENT_UPDATE_PROGRESS, notify_me)
flo = lf.Flow("test")
flo.add(t)
b, fd = p_utils.temporary_flow_detail(be)
e = self._make_engine(flo, flow_detail=fd, backend=be)
e.run()
end_progress = e.storage.get_task_progress("test")
self.assertEqual(1.0, end_progress)
task_uuid = e.storage.get_atom_uuid("test")
td = fd.find(task_uuid)
self.assertEqual(1.0, td.meta['progress'])
self.assertFalse(td.meta['progress_details'])
self.assertEqual(6, len(fired_events))
def setUp(self):
super(EngineTestBase, self).setUp()
self.backend = impl_memory.MemoryBackend(conf={})
def setUp(self):
super(MemoryPersistenceTest, self).setUp()
self._backend = impl_memory.MemoryBackend({})
def tearDown(self):
super(TestProgress, self).tearDown()
with contextlib.closing(impl_memory.MemoryBackend({})) as be:
with contextlib.closing(be.get_connection()) as conn:
conn.clear_all()
