class TestTimeExecution(TestBaseBackend): def setUp(self): self.qsize = 10 self.qtimeout = 0.1 self.mocked_backend = mock.Mock( spec=elasticsearch.ElasticsearchBackend) self.MockedBackendClass = mock.Mock(return_value=self.mocked_backend) self.backend = ThreadedBackend( self.MockedBackendClass, backend_args=('arg1', 'arg2'), backend_kwargs=dict(key1='kwarg1', key2='kwarg2'), queue_maxsize=self.qsize, queue_timeout=self.qtimeout, ) self.backend.bulk_size = self.qsize / 2 self.backend.bulk_timeout = self.qtimeout * 2 settings.configure(backends=[self.backend]) def stop_worker(self): self.backend.worker_limit = 0 time.sleep(self.qtimeout * 2) self.assertEqual(self.backend.thread, None) def resume_worker(self, worker_limit=None, **kwargs): self.backend.worker_limit = worker_limit for key, val in kwargs.items(): if hasattr(self.backend, key): setattr(self.backend, key, val) self.backend.start_worker() def test_thread_name(self): self.assertEqual(self.backend.thread.name, "TimeExecutionThread") def test_backend_args(self): self.MockedBackendClass.assert_called_with('arg1', 'arg2', key1='kwarg1', key2='kwarg2') ThreadedBackend(self.MockedBackendClass) self.MockedBackendClass.assert_called_with() def test_empty_queue(self): time.sleep(2 * self.qtimeout) # ensures queue.get times out self.assertEqual(0, self.backend.fetched_items) def test_decorator(self): now = datetime.now() with freeze_time(now): go() # ensure worker thread catches up time.sleep(2 * self.backend.bulk_timeout) mocked_write = self.mocked_backend.bulk_write self.assertEqual(1, self.backend.fetched_items) mocked_write.assert_called_with([{ 'timestamp': now, 'hostname': SHORT_HOSTNAME, 'name': 'tests.conftest.go', 'value': 0.0, }]) def test_double_start(self): self.assertEqual(0, self.backend.fetched_items) go() time.sleep(2 * self.qtimeout) self.assertEqual(1, self.backend.fetched_items) # try to double start self.backend.start_worker() self.assertEqual(1, self.backend.fetched_items) def test_write_error(self): self.mocked_backend.write.side_effect = RuntimeError('mocked') go() time.sleep(2 * self.qtimeout) def test_queue_congestion(self): # assure worker is stopped self.stop_worker() # fill in the queue for _ in range(self.qsize * 2): go() self.assertTrue(self.backend._queue.full()) self.resume_worker(bulk_timeout=self.qtimeout) # wait until all metrics are picked up time.sleep(self.qsize * self.qtimeout) # check that metrics in the queue were sent with bulk_write calls call_args_list = self.mocked_backend.bulk_write.call_args_list time.sleep(2 * self.qtimeout) self.assertEqual(self.qsize, sum(len(args[0]) for args, _ in call_args_list)) def test_worker_sends_remainder(self): self.stop_worker() self.mocked_backend.bulk_write.side_effect = RuntimeError('mock') loops_count = 3 self.assertTrue(loops_count < self.backend.bulk_size) for _ in range(loops_count): go() self.backend.worker_limit = loops_count self.backend.worker() self.assertEqual(loops_count, self.backend.fetched_items) mocked_bulk_write = self.mocked_backend.bulk_write mocked_bulk_write.assert_called_once() time.sleep(self.qtimeout * 2) self.assertEqual(loops_count, len(mocked_bulk_write.call_args[0][0])) def test_worker_error(self): self.assertFalse(self.backend.thread is None) # simulate TypeError in queue.get with mock.patch.object(self.backend._queue, 'get', side_effect=TypeError): # ensure worker loop repeat time.sleep(2 * self.qtimeout) # assert thread stopped self.assertTrue(self.backend.thread is None) def test_producer_in_another_process(self): # assure worker is stopped self.stop_worker() # fill in the queue process = Process(target=go) process.start() process.join() # check the queue contains the item self.assertEqual(self.backend._queue.qsize(), 1) def test_flush_metrics_when_parent_process_not_alive(self): self.stop_worker() loops = 3 with mock.patch.object(self.backend, 'parent_thread', spec=Thread) as parent_thread: parent_thread.is_alive.return_value = False for _ in range(loops): go() #: do not allow flush metrics before checking if parent_thread is alive self.backend.worker_limit = loops + 1 self.backend.worker() mocked_bulk_write = self.mocked_backend.bulk_write mocked_bulk_write.assert_called_once() self.assertEqual(loops, len(mocked_bulk_write.call_args[0][0]))
class TestTimeExecution(TestBaseBackend): def setUp(self): self.qsize = 10 self.qtimeout = 0.1 self.mocked_backend = mock.Mock(spec=elasticsearch.ElasticsearchBackend) self.MockedBackendClass = mock.Mock(return_value=self.mocked_backend) self.backend = ThreadedBackend( self.MockedBackendClass, backend_args=('arg1', 'arg2'), backend_kwargs=dict(key1='kwarg1', key2='kwarg2'), queue_maxsize=self.qsize, queue_timeout=self.qtimeout, ) self.backend.bulk_size = self.qsize / 2 self.backend.bulk_timeout = self.qtimeout * 2 settings.configure(backends=[self.backend]) def stop_worker(self): self.backend.worker_limit = 0 time.sleep(self.qtimeout * 2) self.assertEqual(self.backend.thread, None) def resume_worker(self, worker_limit=None, **kwargs): self.backend.worker_limit = worker_limit for key, val in kwargs.items(): if hasattr(self.backend, key): setattr(self.backend, key, val) self.backend.start_worker() def test_thread_name(self): self.assertEquals(self.backend.thread.name, "TimeExecutionThread") def test_backend_args(self): self.MockedBackendClass.assert_called_with('arg1', 'arg2', key1='kwarg1', key2='kwarg2') ThreadedBackend(self.MockedBackendClass) self.MockedBackendClass.assert_called_with() def test_empty_queue(self): time.sleep(2 * self.qtimeout) # ensures queue.get times out self.assertEqual(0, self.backend.fetched_items) def test_decorator(self): with freeze_time('2016-08-01 00:00:00'): go() # ensure worker thread catches up time.sleep(2 * self.backend.bulk_timeout) mocked_write = self.mocked_backend.bulk_write self.assertEqual(1, self.backend.fetched_items) mocked_write.assert_called_with([{ 'timestamp': datetime(2016, 8, 1, 0, 0), 'hostname': SHORT_HOSTNAME, 'name': 'tests.conftest.go', 'value': 0.0, }]) def test_double_start(self): self.assertEqual(0, self.backend.fetched_items) go() time.sleep(2 * self.qtimeout) self.assertEqual(1, self.backend.fetched_items) # try to double start self.backend.start_worker() self.assertEqual(1, self.backend.fetched_items) def test_write_error(self): self.mocked_backend.write.side_effect = RuntimeError('mocked') go() time.sleep(2 * self.qtimeout) def test_queue_congestion(self): # assure worker is stopped self.stop_worker() # fill in the queue for _ in range(self.qsize * 2): go() self.assertTrue(self.backend._queue.full()) self.resume_worker(bulk_timeout=self.qtimeout) # wait until all metrics are picked up time.sleep(self.qsize * self.qtimeout) # check that metrics in the queue were sent with bulk_write calls call_args_list = self.mocked_backend.bulk_write.call_args_list time.sleep(2 * self.qtimeout) self.assertEqual( self.qsize, sum(len(args[0]) for args, _ in call_args_list) ) def test_worker_sends_remainder(self): self.stop_worker() self.mocked_backend.bulk_write.side_effect = RuntimeError('mock') loops_count = 3 self.assertTrue(loops_count < self.backend.bulk_size) for _ in range(loops_count): go() self.backend.worker_limit = loops_count self.backend.worker() self.assertEqual(loops_count, self.backend.fetched_items) mocked_bulk_write = self.mocked_backend.bulk_write mocked_bulk_write.assert_called_once() time.sleep(self.qtimeout * 2) self.assertEqual( loops_count, len(mocked_bulk_write.call_args[0][0]) ) def test_worker_error(self): self.assertFalse(self.backend.thread is None) # simulate TypeError in queue.get with mock.patch.object(self.backend._queue, 'get', side_effect=TypeError): # ensure worker loop repeat time.sleep(2 * self.qtimeout) # assert thread stopped self.assertTrue(self.backend.thread is None)
class TestTimeExecution(TestBaseBackend): def setUp(self): self.qsize = 10 self.qtimeout = 0.1 self.mocked_backend = mock.Mock( spec=elasticsearch.ElasticsearchBackend) self.MockedBackendClass = mock.Mock(return_value=self.mocked_backend) self.backend = ThreadedBackend( self.MockedBackendClass, backend_args=('arg1', 'arg2'), backend_kwargs=dict(key1='kwarg1', key2='kwarg2'), queue_maxsize=self.qsize, queue_timeout=self.qtimeout, ) self.backend.bulk_size = self.qsize / 2 self.backend.bulk_timeout = self.qtimeout * 2 settings.configure(backends=[self.backend]) def stop_worker(self): self.backend.worker_limit = 0 time.sleep(self.qtimeout * 2) self.assertEqual(self.backend.thread, None) def resume_worker(self, worker_limit=None, **kwargs): self.backend.worker_limit = worker_limit for key, val in kwargs.items(): if hasattr(self.backend, key): setattr(self.backend, key, val) self.backend.start_worker() def test_backend_args(self): self.MockedBackendClass.assert_called_with('arg1', 'arg2', key1='kwarg1', key2='kwarg2') ThreadedBackend(self.MockedBackendClass) self.MockedBackendClass.assert_called_with() def test_empty_queue(self): time.sleep(2 * self.qtimeout) # ensures queue.get times out self.assertEqual(0, self.backend.fetched_items) def test_decorator(self): with freeze_time('2016-08-01 00:00:00'): go() # ensure worker thread catches up time.sleep(2 * self.backend.bulk_timeout) mocked_write = self.mocked_backend.bulk_write self.assertEqual(1, self.backend.fetched_items) mocked_write.assert_called_with([{ 'timestamp': datetime(2016, 8, 1, 0, 0), 'hostname': SHORT_HOSTNAME, 'name': 'tests.conftest.go', 'value': 0.0, }]) def test_double_start(self): self.assertEqual(0, self.backend.fetched_items) go() time.sleep(2 * self.qtimeout) self.assertEqual(1, self.backend.fetched_items) # try to double start self.backend.start_worker() self.assertEqual(1, self.backend.fetched_items) def test_write_error(self): self.mocked_backend.write.side_effect = RuntimeError('mocked') go() time.sleep(2 * self.qtimeout) def test_queue_congestion(self): # assure worker is stopped self.stop_worker() # fill in the queue for _ in range(self.qsize * 2): go() self.assertTrue(self.backend._queue.full()) self.resume_worker(bulk_timeout=self.qtimeout) # wait until all metrics are picked up time.sleep(self.qsize * self.qtimeout) # check that metrics in the queue were sent with bulk_write calls call_args_list = self.mocked_backend.bulk_write.call_args_list self.assertEqual(self.qsize, sum(len(args[0]) for args, _ in call_args_list)) def test_worker_sends_remainder(self): self.stop_worker() self.mocked_backend.bulk_write.side_effect = RuntimeError('mock') loops_count = 3 self.assertTrue(loops_count < self.backend.bulk_size) for _ in range(loops_count): go() self.backend.worker_limit = loops_count self.backend.worker() self.assertEqual(loops_count, self.backend.fetched_items) mocked_bulk_write = self.mocked_backend.bulk_write mocked_bulk_write.assert_called_once() self.assertEqual(loops_count, len(mocked_bulk_write.call_args[0][0])) def test_worker_error(self): self.assertFalse(self.backend.thread is None) # simulate TypeError in queue.get with mock.patch.object(self.backend._queue, 'get', side_effect=TypeError): # ensure worker loop repeat time.sleep(2 * self.qtimeout) # assert thread stopped self.assertTrue(self.backend.thread is None)