def test_index_conflict_with_existing_success(self, datetime_mock): expected_dt = datetime.datetime(2020, 6, 1, 18, 43, 24, 123456, None) datetime_mock.utcnow.return_value = expected_dt x = ElasticsearchBackend(app=self.app) x._server = Mock() x._server.index.side_effect = [ exceptions.ConflictError(409, "concurrent update", {}) ] x._server.get.return_value = { 'found': True, '_source': { 'result': """{"status":"SUCCESS","result":42}""" }, '_seq_no': 2, '_primary_term': 1, } x._server.update.return_value = {'result': 'updated'} x.set(sentinel.task_id, sentinel.result, sentinel.state) assert x._server.get.call_count == 1 x._server.index.assert_called_once_with( id=sentinel.task_id, index=x.index, doc_type=x.doc_type, body={ 'result': sentinel.result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' }, params={'op_type': 'create'}, ) x._server.update.assert_not_called()
def test_exception_safe_to_retry(self): x = ElasticsearchBackend(app=self.app) assert not x.exception_safe_to_retry(Exception("failed")) assert not x.exception_safe_to_retry(BaseException("failed")) assert x.exception_safe_to_retry(exceptions.ConflictError(409, "concurrent update", {})) assert x.exception_safe_to_retry(exceptions.ConnectionError(503, "service unavailable", {})) assert x.exception_safe_to_retry(exceptions.TransportError(429, "too many requests", {})) assert not x.exception_safe_to_retry(exceptions.NotFoundError(404, "not found", {}))
def test_index_conflict_with_ready_state_on_backend_without_state( self, datetime_mock): """Even if the backend already have a ready state saved (FAILURE in this test case) as we are calling ElasticsearchBackend.set directly, it does not have state, so it cannot protect overriding a ready state by any other state. As a result, server.update will be called no matter what. """ expected_dt = datetime.datetime(2020, 6, 1, 18, 43, 24, 123456, None) datetime_mock.utcnow.return_value = expected_dt x = ElasticsearchBackend(app=self.app) x._server = Mock() x._server.index.side_effect = [ exceptions.ConflictError(409, "concurrent update", {}) ] x._server.get.return_value = { 'found': True, '_source': { 'result': """{"status":"FAILURE","result":{"exc_type":"Exception","exc_message":["failed"],"exc_module":"builtins"}}""" }, '_seq_no': 2, '_primary_term': 1, } x._server.update.return_value = {'result': 'updated'} x.set(sentinel.task_id, sentinel.result) assert x._server.get.call_count == 1 x._server.index.assert_called_once_with( id=sentinel.task_id, index=x.index, doc_type=x.doc_type, body={ 'result': sentinel.result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' }, params={'op_type': 'create'}, ) x._server.update.assert_called_once_with( id=sentinel.task_id, index=x.index, doc_type=x.doc_type, body={ 'doc': { 'result': sentinel.result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' } }, params={ 'if_seq_no': 2, 'if_primary_term': 1 })
def test_backend_concurrent_update(self, base_datetime_mock, es_datetime_mock): expected_dt = datetime.datetime(2020, 6, 1, 18, 43, 24, 123456, None) es_datetime_mock.utcnow.return_value = expected_dt expected_done_dt = datetime.datetime(2020, 6, 1, 18, 45, 34, 654321, None) base_datetime_mock.utcnow.return_value = expected_done_dt self.app.conf.result_backend_always_retry, prev = True, self.app.conf.result_backend_always_retry try: x = ElasticsearchBackend(app=self.app) task_id = str(sentinel.task_id) encoded_task_id = bytes_to_str(x.get_key_for_task(task_id)) result = str(sentinel.result) sleep_mock = Mock() x._sleep = sleep_mock x._server = Mock() x._server.index.side_effect = exceptions.ConflictError( 409, "concurrent update", {}) x._server.get.side_effect = [ { 'found': True, '_source': { 'result': """{"status":"RETRY","result":{"exc_type":"Exception","exc_message":["failed"],"exc_module":"builtins"}}""" }, '_seq_no': 2, '_primary_term': 1, }, { 'found': True, '_source': { 'result': """{"status":"RETRY","result":{"exc_type":"Exception","exc_message":["failed"],"exc_module":"builtins"}}""" }, '_seq_no': 2, '_primary_term': 1, }, { 'found': True, '_source': { 'result': """{"status":"FAILURE","result":{"exc_type":"Exception","exc_message":["failed"],"exc_module":"builtins"}}""" }, '_seq_no': 3, '_primary_term': 1, }, { 'found': True, '_source': { 'result': """{"status":"FAILURE","result":{"exc_type":"Exception","exc_message":["failed"],"exc_module":"builtins"}}""" }, '_seq_no': 3, '_primary_term': 1, }, ] x._server.update.side_effect = [{ 'result': 'noop' }, { 'result': 'updated' }] result_meta = x._get_result_meta(result, states.SUCCESS, None, None) result_meta['task_id'] = bytes_to_str(task_id) expected_result = x.encode(result_meta) x.store_result(task_id, result, states.SUCCESS) x._server.index.assert_has_calls([ call(id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' }, params={'op_type': 'create'}), call(id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' }, params={'op_type': 'create'}), ]) x._server.update.assert_has_calls([ call(id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'doc': { 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' } }, params={ 'if_seq_no': 2, 'if_primary_term': 1 }), call(id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'doc': { 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' } }, params={ 'if_seq_no': 3, 'if_primary_term': 1 }), ]) assert sleep_mock.call_count == 1 finally: self.app.conf.result_backend_always_retry = prev
def test_backend_index_corrupted_conflicting_document( self, base_datetime_mock, es_datetime_mock): expected_dt = datetime.datetime(2020, 6, 1, 18, 43, 24, 123456, None) es_datetime_mock.utcnow.return_value = expected_dt expected_done_dt = datetime.datetime(2020, 6, 1, 18, 45, 34, 654321, None) base_datetime_mock.utcnow.return_value = expected_done_dt # self.app.conf.result_backend_always_retry, prev = True, self.app.conf.result_backend_always_retry # try: x = ElasticsearchBackend(app=self.app) task_id = str(sentinel.task_id) encoded_task_id = bytes_to_str(x.get_key_for_task(task_id)) result = str(sentinel.result) sleep_mock = Mock() x._sleep = sleep_mock x._server = Mock() x._server.index.side_effect = [ exceptions.ConflictError(409, "concurrent update", {}) ] x._server.update.side_effect = [{'result': 'updated'}] x._server.get.return_value = { 'found': True, '_source': {}, '_seq_no': 2, '_primary_term': 1, } result_meta = x._get_result_meta(result, states.SUCCESS, None, None) result_meta['task_id'] = bytes_to_str(task_id) expected_result = x.encode(result_meta) x.store_result(task_id, result, states.SUCCESS) x._server.index.assert_called_once_with( id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' }, params={'op_type': 'create'}) x._server.update.assert_called_once_with( id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'doc': { 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' } }, params={ 'if_primary_term': 1, 'if_seq_no': 2 }) sleep_mock.assert_not_called()
def test_backend_index_conflicting_document_removed_not_throwing( self, base_datetime_mock, es_datetime_mock): expected_dt = datetime.datetime(2020, 6, 1, 18, 43, 24, 123456, None) es_datetime_mock.utcnow.return_value = expected_dt expected_done_dt = datetime.datetime(2020, 6, 1, 18, 45, 34, 654321, None) base_datetime_mock.utcnow.return_value = expected_done_dt self.app.conf.result_backend_always_retry, prev = True, self.app.conf.result_backend_always_retry try: x = ElasticsearchBackend(app=self.app) task_id = str(sentinel.task_id) encoded_task_id = bytes_to_str(x.get_key_for_task(task_id)) result = str(sentinel.result) sleep_mock = Mock() x._sleep = sleep_mock x._server = Mock() x._server.index.side_effect = [ exceptions.ConflictError(409, "concurrent update", {}), { 'result': 'created' } ] x._server.get.side_effect = [ { 'found': True, '_source': { 'result': _RESULT_RETRY }, '_seq_no': 2, '_primary_term': 1, }, { '_index': 'celery', '_type': '_doc', '_id': 'toto', 'found': False }, ] result_meta = x._get_result_meta(result, states.SUCCESS, None, None) result_meta['task_id'] = bytes_to_str(task_id) expected_result = x.encode(result_meta) x.store_result(task_id, result, states.SUCCESS) x._server.index.assert_has_calls([ call(id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' }, params={'op_type': 'create'}), call(id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' }, params={'op_type': 'create'}), ]) x._server.update.assert_not_called() sleep_mock.assert_not_called() finally: self.app.conf.result_backend_always_retry = prev