def make_components(self, name='testing', wait_timeout=0.1):
client = fake_client.FakeClient()
persistence = impl_memory.MemoryBackend()
board = impl_zookeeper.ZookeeperJobBoard(name, {},
client=client,
persistence=persistence)
conductor = stc.SingleThreadedConductor(name, board, persistence,
wait_timeout=wait_timeout)
return self.ComponentBundle(board, client, persistence, conductor)
def test_deregister(self):
"""Verify that register and deregister don't blow up"""
with contextlib.closing(impl_memory.MemoryBackend()) as be:
flow = lf.Flow("test")
flow.add(SleepyTask("test-1", sleep_for=0.1))
(lb, fd) = persistence_utils.temporary_flow_detail(be)
e = self._make_engine(flow, fd, be)
l = timing.DurationListener(e)
l.register()
l.deregister()
def make_components(self):
client = fake_client.FakeClient()
persistence = impl_memory.MemoryBackend()
board = impl_zookeeper.ZookeeperJobBoard('testing', {},
client=client,
persistence=persistence)
conductor_kwargs = self.conductor_kwargs.copy()
conductor_kwargs['persistence'] = persistence
conductor = backends.fetch(self.kind, 'testing', board,
**conductor_kwargs)
return ComponentBundle(board, client, persistence, conductor)
def test_flow_duration(self):
with contextlib.closing(impl_memory.MemoryBackend()) as be:
flow = lf.Flow("test")
flow.add(SleepyTask("test-1", sleep_for=0.1))
(lb, fd) = persistence_utils.temporary_flow_detail(be)
e = self._make_engine(flow, fd, be)
with timing.DurationListener(e):
e.run()
self.assertIsNotNone(fd)
self.assertIsNotNone(fd.meta)
self.assertIn('duration', fd.meta)
self.assertGreaterEqual(0.1, fd.meta['duration'])
def test_storage_progress(self):
with contextlib.closing(impl_memory.MemoryBackend({})) as be:
flo = lf.Flow("test")
flo.add(ProgressTask("test", 3))
b, fd = p_utils.temporary_flow_detail(be)
e = self._make_engine(flo, flow_detail=fd, backend=be)
e.run()
end_progress = e.storage.get_task_progress("test")
self.assertEqual(1.0, end_progress)
task_uuid = e.storage.get_task_uuid("test")
td = fd.find(task_uuid)
self.assertEqual(1.0, td.meta['progress'])
self.assertFalse(td.meta['progress_details'])
def test_bad_factory(self):
persistence = impl_memory.MemoryBackend()
client = fake_client.FakeClient()
board = impl_zookeeper.ZookeeperJobBoard('testing', {},
client=client,
persistence=persistence)
self.assertRaises(ValueError,
backends.fetch,
'nonblocking',
'testing',
board,
persistence=persistence,
executor_factory='testing')
def test_record_ending_exception(self, mocked_warn):
with contextlib.closing(impl_memory.MemoryBackend()) as be:
flow = lf.Flow("test")
flow.add(test_utils.TaskNoRequiresNoReturns("test-1"))
(lb, fd) = persistence_utils.temporary_flow_detail(be)
e = self._make_engine(flow, fd, be)
timing_listener = timing.TimingListener(e)
with mock.patch.object(timing_listener._engine.storage,
'update_atom_metadata') as mocked_uam:
mocked_uam.side_effect = exc.StorageFailure('Woot!')
with timing_listener:
e.run()
mocked_warn.assert_called_once_with(mock.ANY, mock.ANY, 'test-1',
exc_info=True)
def test_duration(self):
with contextlib.closing(impl_memory.MemoryBackend({})) as be:
flo = lf.Flow("test")
flo.add(SleepyTask("test-1", sleep_for=0.1))
(lb, fd) = p_utils.temporary_flow_detail(be)
e = self.make_engine(flo, fd, be)
with timing.TimingListener(e):
e.run()
t_uuid = e.storage.get_task_uuid("test-1")
td = fd.find(t_uuid)
self.assertIsNotNone(td)
self.assertIsNotNone(td.meta)
self.assertIn('duration', td.meta)
self.assertGreaterEqual(0.1, td.meta['duration'])
def __init__(self, flow_detail, backend=None, scope_fetcher=None):
self._result_mappings = {}
self._reverse_mapping = {}
if backend is None:
# Err on the likely-hood that most people don't make there
# objects able to be deepcopyable (resources, locks and such
# can't be deepcopied)...
backend = impl_memory.MemoryBackend({'deep_copy': False})
with contextlib.closing(backend.get_connection()) as conn:
conn.update_flow_details(flow_detail, ignore_missing=True)
self._backend = backend
self._flowdetail = flow_detail
self._transients = {}
self._injected_args = {}
self._lock = fasteners.ReaderWriterLock()
self._ensure_matchers = [
((task.BaseTask, ), (models.TaskDetail, 'Task')),
((retry.Retry, ), (models.RetryDetail, 'Retry')),
]
if scope_fetcher is None:
scope_fetcher = lambda atom_name: None
self._scope_fetcher = scope_fetcher
# NOTE(imelnikov): failure serialization looses information,
# so we cache failures here, in atom name -> failure mapping.
self._failures = {}
for ad in self._flowdetail:
fail_cache = {}
if ad.failure is not None:
fail_cache[states.EXECUTE] = ad.failure
if ad.revert_failure is not None:
fail_cache[states.REVERT] = ad.revert_failure
self._failures[ad.name] = fail_cache
self._atom_name_to_uuid = dict(
(ad.name, ad.uuid) for ad in self._flowdetail)
try:
source, _clone = self._atomdetail_by_name(
self.injector_name, expected_type=models.TaskDetail)
except exceptions.NotFound:
pass
else:
names_iter = six.iterkeys(source.results)
self._set_result_mapping(source.name,
dict((name, name) for name in names_iter))
def test_dual_storage_progress(self):
fired_events = []
def notify_me(task, event_data, progress):
fired_events.append(progress)
with contextlib.closing(impl_memory.MemoryBackend({})) as be:
t = ProgressTask("test", 5)
t.bind('update_progress', notify_me)
flo = lf.Flow("test")
flo.add(t)
b, fd = p_utils.temporary_flow_detail(be)
e = self._make_engine(flo, flow_detail=fd, backend=be)
e.run()
end_progress = e.storage.get_task_progress("test")
self.assertEqual(1.0, end_progress)
task_uuid = e.storage.get_task_uuid("test")
td = fd.find(task_uuid)
self.assertEqual(1.0, td.meta['progress'])
self.assertFalse(td.meta['progress_details'])
self.assertEqual(6, len(fired_events))
def test_dual_storage_progress(self):
fired_events = []
def notify_me(event_type, details):
fired_events.append(details.pop('progress'))
with contextlib.closing(impl_memory.MemoryBackend({})) as be:
t = ProgressTask("test", 5)
t.notifier.register(task.EVENT_UPDATE_PROGRESS, notify_me)
flo = lf.Flow("test")
flo.add(t)
b, fd = p_utils.temporary_flow_detail(be)
e = self._make_engine(flo, flow_detail=fd, backend=be)
e.run()
end_progress = e.storage.get_task_progress("test")
self.assertEqual(1.0, end_progress)
task_uuid = e.storage.get_atom_uuid("test")
td = fd.find(task_uuid)
self.assertEqual(1.0, td.meta['progress'])
self.assertFalse(td.meta['progress_details'])
self.assertEqual(6, len(fired_events))
def setUp(self):
super(EngineTestBase, self).setUp()
self.values = []
self.backend = impl_memory.MemoryBackend(conf={})
def setUp(self):
super(MemoryPersistenceTest, self).setUp()
self._backend = impl_memory.MemoryBackend({})
def setUp(self):
super(StorageTest, self).setUp()
self.backend = impl_memory.MemoryBackend(conf={})
def setUp(self):
self._backend = impl_memory.MemoryBackend({})
def tearDown(self):
super(TestProgress, self).tearDown()
with contextlib.closing(impl_memory.MemoryBackend({})) as be:
with contextlib.closing(be.get_connection()) as conn:
conn.clear_all()
