def make_components(self, name='testing', wait_timeout=0.1): client = fake_client.FakeClient() persistence = impl_memory.MemoryBackend() board = impl_zookeeper.ZookeeperJobBoard(name, {}, client=client, persistence=persistence) conductor = stc.SingleThreadedConductor(name, board, persistence, wait_timeout=wait_timeout) return self.ComponentBundle(board, client, persistence, conductor)
def test_deregister(self): """Verify that register and deregister don't blow up""" with contextlib.closing(impl_memory.MemoryBackend()) as be: flow = lf.Flow("test") flow.add(SleepyTask("test-1", sleep_for=0.1)) (lb, fd) = persistence_utils.temporary_flow_detail(be) e = self._make_engine(flow, fd, be) l = timing.DurationListener(e) l.register() l.deregister()
def make_components(self): client = fake_client.FakeClient() persistence = impl_memory.MemoryBackend() board = impl_zookeeper.ZookeeperJobBoard('testing', {}, client=client, persistence=persistence) conductor_kwargs = self.conductor_kwargs.copy() conductor_kwargs['persistence'] = persistence conductor = backends.fetch(self.kind, 'testing', board, **conductor_kwargs) return ComponentBundle(board, client, persistence, conductor)
def test_flow_duration(self): with contextlib.closing(impl_memory.MemoryBackend()) as be: flow = lf.Flow("test") flow.add(SleepyTask("test-1", sleep_for=0.1)) (lb, fd) = persistence_utils.temporary_flow_detail(be) e = self._make_engine(flow, fd, be) with timing.DurationListener(e): e.run() self.assertIsNotNone(fd) self.assertIsNotNone(fd.meta) self.assertIn('duration', fd.meta) self.assertGreaterEqual(0.1, fd.meta['duration'])
def test_storage_progress(self): with contextlib.closing(impl_memory.MemoryBackend({})) as be: flo = lf.Flow("test") flo.add(ProgressTask("test", 3)) b, fd = p_utils.temporary_flow_detail(be) e = self._make_engine(flo, flow_detail=fd, backend=be) e.run() end_progress = e.storage.get_task_progress("test") self.assertEqual(1.0, end_progress) task_uuid = e.storage.get_task_uuid("test") td = fd.find(task_uuid) self.assertEqual(1.0, td.meta['progress']) self.assertFalse(td.meta['progress_details'])
def test_bad_factory(self): persistence = impl_memory.MemoryBackend() client = fake_client.FakeClient() board = impl_zookeeper.ZookeeperJobBoard('testing', {}, client=client, persistence=persistence) self.assertRaises(ValueError, backends.fetch, 'nonblocking', 'testing', board, persistence=persistence, executor_factory='testing')
def test_record_ending_exception(self, mocked_warn): with contextlib.closing(impl_memory.MemoryBackend()) as be: flow = lf.Flow("test") flow.add(test_utils.TaskNoRequiresNoReturns("test-1")) (lb, fd) = persistence_utils.temporary_flow_detail(be) e = self._make_engine(flow, fd, be) timing_listener = timing.TimingListener(e) with mock.patch.object(timing_listener._engine.storage, 'update_atom_metadata') as mocked_uam: mocked_uam.side_effect = exc.StorageFailure('Woot!') with timing_listener: e.run() mocked_warn.assert_called_once_with(mock.ANY, mock.ANY, 'test-1', exc_info=True)
def test_duration(self): with contextlib.closing(impl_memory.MemoryBackend({})) as be: flo = lf.Flow("test") flo.add(SleepyTask("test-1", sleep_for=0.1)) (lb, fd) = p_utils.temporary_flow_detail(be) e = self.make_engine(flo, fd, be) with timing.TimingListener(e): e.run() t_uuid = e.storage.get_task_uuid("test-1") td = fd.find(t_uuid) self.assertIsNotNone(td) self.assertIsNotNone(td.meta) self.assertIn('duration', td.meta) self.assertGreaterEqual(0.1, td.meta['duration'])
def __init__(self, flow_detail, backend=None, scope_fetcher=None): self._result_mappings = {} self._reverse_mapping = {} if backend is None: # Err on the likely-hood that most people don't make there # objects able to be deepcopyable (resources, locks and such # can't be deepcopied)... backend = impl_memory.MemoryBackend({'deep_copy': False}) with contextlib.closing(backend.get_connection()) as conn: conn.update_flow_details(flow_detail, ignore_missing=True) self._backend = backend self._flowdetail = flow_detail self._transients = {} self._injected_args = {} self._lock = fasteners.ReaderWriterLock() self._ensure_matchers = [ ((task.BaseTask, ), (models.TaskDetail, 'Task')), ((retry.Retry, ), (models.RetryDetail, 'Retry')), ] if scope_fetcher is None: scope_fetcher = lambda atom_name: None self._scope_fetcher = scope_fetcher # NOTE(imelnikov): failure serialization looses information, # so we cache failures here, in atom name -> failure mapping. self._failures = {} for ad in self._flowdetail: fail_cache = {} if ad.failure is not None: fail_cache[states.EXECUTE] = ad.failure if ad.revert_failure is not None: fail_cache[states.REVERT] = ad.revert_failure self._failures[ad.name] = fail_cache self._atom_name_to_uuid = dict( (ad.name, ad.uuid) for ad in self._flowdetail) try: source, _clone = self._atomdetail_by_name( self.injector_name, expected_type=models.TaskDetail) except exceptions.NotFound: pass else: names_iter = six.iterkeys(source.results) self._set_result_mapping(source.name, dict((name, name) for name in names_iter))
def test_dual_storage_progress(self): fired_events = [] def notify_me(task, event_data, progress): fired_events.append(progress) with contextlib.closing(impl_memory.MemoryBackend({})) as be: t = ProgressTask("test", 5) t.bind('update_progress', notify_me) flo = lf.Flow("test") flo.add(t) b, fd = p_utils.temporary_flow_detail(be) e = self._make_engine(flo, flow_detail=fd, backend=be) e.run() end_progress = e.storage.get_task_progress("test") self.assertEqual(1.0, end_progress) task_uuid = e.storage.get_task_uuid("test") td = fd.find(task_uuid) self.assertEqual(1.0, td.meta['progress']) self.assertFalse(td.meta['progress_details']) self.assertEqual(6, len(fired_events))
def test_dual_storage_progress(self): fired_events = [] def notify_me(event_type, details): fired_events.append(details.pop('progress')) with contextlib.closing(impl_memory.MemoryBackend({})) as be: t = ProgressTask("test", 5) t.notifier.register(task.EVENT_UPDATE_PROGRESS, notify_me) flo = lf.Flow("test") flo.add(t) b, fd = p_utils.temporary_flow_detail(be) e = self._make_engine(flo, flow_detail=fd, backend=be) e.run() end_progress = e.storage.get_task_progress("test") self.assertEqual(1.0, end_progress) task_uuid = e.storage.get_atom_uuid("test") td = fd.find(task_uuid) self.assertEqual(1.0, td.meta['progress']) self.assertFalse(td.meta['progress_details']) self.assertEqual(6, len(fired_events))
def setUp(self): super(EngineTestBase, self).setUp() self.values = [] self.backend = impl_memory.MemoryBackend(conf={})
def setUp(self): super(MemoryPersistenceTest, self).setUp() self._backend = impl_memory.MemoryBackend({})
def setUp(self): super(StorageTest, self).setUp() self.backend = impl_memory.MemoryBackend(conf={})
def setUp(self): self._backend = impl_memory.MemoryBackend({})
def tearDown(self): super(TestProgress, self).tearDown() with contextlib.closing(impl_memory.MemoryBackend({})) as be: with contextlib.closing(be.get_connection()) as conn: conn.clear_all()