def test_encode_none_as_json(self): self.app.conf.elasticsearch_save_meta_as_text, prev = False, self.app.conf.elasticsearch_save_meta_as_text try: x = ElasticsearchBackend(app=self.app) result_meta = x._get_result_meta(None, states.SUCCESS, None, None) assert x.encode(result_meta) == result_meta finally: self.app.conf.elasticsearch_save_meta_as_text = prev
def test_decode_encoded_from_json(self): self.app.conf.elasticsearch_save_meta_as_text, prev = False, self.app.conf.elasticsearch_save_meta_as_text try: x = ElasticsearchBackend(app=self.app) result_meta = x._get_result_meta({'solution': 42}, states.SUCCESS, None, None) assert x.decode(x.encode(result_meta)) == result_meta finally: self.app.conf.elasticsearch_save_meta_as_text = prev
def test_decode_encoded_exception_as_json(self): self.app.conf.elasticsearch_save_meta_as_text, prev = False, self.app.conf.elasticsearch_save_meta_as_text try: x = ElasticsearchBackend(app=self.app) try: raise Exception("failed") except Exception as exc: einfo = ExceptionInfo() result_meta = x._get_result_meta( x.encode_result(exc, states.FAILURE), states.FAILURE, einfo.traceback, None) assert x.decode(x.encode(result_meta)) == result_meta finally: self.app.conf.elasticsearch_save_meta_as_text = prev
def test_backend_concurrent_update(self, base_datetime_mock, es_datetime_mock): expected_dt = datetime.datetime(2020, 6, 1, 18, 43, 24, 123456, None) es_datetime_mock.utcnow.return_value = expected_dt expected_done_dt = datetime.datetime(2020, 6, 1, 18, 45, 34, 654321, None) base_datetime_mock.utcnow.return_value = expected_done_dt self.app.conf.result_backend_always_retry, prev = True, self.app.conf.result_backend_always_retry try: x = ElasticsearchBackend(app=self.app) task_id = str(sentinel.task_id) encoded_task_id = bytes_to_str(x.get_key_for_task(task_id)) result = str(sentinel.result) sleep_mock = Mock() x._sleep = sleep_mock x._server = Mock() x._server.index.side_effect = exceptions.ConflictError( 409, "concurrent update", {}) x._server.get.side_effect = [ { 'found': True, '_source': { 'result': """{"status":"RETRY","result":{"exc_type":"Exception","exc_message":["failed"],"exc_module":"builtins"}}""" }, '_seq_no': 2, '_primary_term': 1, }, { 'found': True, '_source': { 'result': """{"status":"RETRY","result":{"exc_type":"Exception","exc_message":["failed"],"exc_module":"builtins"}}""" }, '_seq_no': 2, '_primary_term': 1, }, { 'found': True, '_source': { 'result': """{"status":"FAILURE","result":{"exc_type":"Exception","exc_message":["failed"],"exc_module":"builtins"}}""" }, '_seq_no': 3, '_primary_term': 1, }, { 'found': True, '_source': { 'result': """{"status":"FAILURE","result":{"exc_type":"Exception","exc_message":["failed"],"exc_module":"builtins"}}""" }, '_seq_no': 3, '_primary_term': 1, }, ] x._server.update.side_effect = [{ 'result': 'noop' }, { 'result': 'updated' }] result_meta = x._get_result_meta(result, states.SUCCESS, None, None) result_meta['task_id'] = bytes_to_str(task_id) expected_result = x.encode(result_meta) x.store_result(task_id, result, states.SUCCESS) x._server.index.assert_has_calls([ call(id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' }, params={'op_type': 'create'}), call(id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' }, params={'op_type': 'create'}), ]) x._server.update.assert_has_calls([ call(id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'doc': { 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' } }, params={ 'if_seq_no': 2, 'if_primary_term': 1 }), call(id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'doc': { 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' } }, params={ 'if_seq_no': 3, 'if_primary_term': 1 }), ]) assert sleep_mock.call_count == 1 finally: self.app.conf.result_backend_always_retry = prev
def test_backend_index_corrupted_conflicting_document( self, base_datetime_mock, es_datetime_mock): expected_dt = datetime.datetime(2020, 6, 1, 18, 43, 24, 123456, None) es_datetime_mock.utcnow.return_value = expected_dt expected_done_dt = datetime.datetime(2020, 6, 1, 18, 45, 34, 654321, None) base_datetime_mock.utcnow.return_value = expected_done_dt # self.app.conf.result_backend_always_retry, prev = True, self.app.conf.result_backend_always_retry # try: x = ElasticsearchBackend(app=self.app) task_id = str(sentinel.task_id) encoded_task_id = bytes_to_str(x.get_key_for_task(task_id)) result = str(sentinel.result) sleep_mock = Mock() x._sleep = sleep_mock x._server = Mock() x._server.index.side_effect = [ exceptions.ConflictError(409, "concurrent update", {}) ] x._server.update.side_effect = [{'result': 'updated'}] x._server.get.return_value = { 'found': True, '_source': {}, '_seq_no': 2, '_primary_term': 1, } result_meta = x._get_result_meta(result, states.SUCCESS, None, None) result_meta['task_id'] = bytes_to_str(task_id) expected_result = x.encode(result_meta) x.store_result(task_id, result, states.SUCCESS) x._server.index.assert_called_once_with( id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' }, params={'op_type': 'create'}) x._server.update.assert_called_once_with( id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'doc': { 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' } }, params={ 'if_primary_term': 1, 'if_seq_no': 2 }) sleep_mock.assert_not_called()
def test_backend_index_conflicting_document_removed_not_throwing( self, base_datetime_mock, es_datetime_mock): expected_dt = datetime.datetime(2020, 6, 1, 18, 43, 24, 123456, None) es_datetime_mock.utcnow.return_value = expected_dt expected_done_dt = datetime.datetime(2020, 6, 1, 18, 45, 34, 654321, None) base_datetime_mock.utcnow.return_value = expected_done_dt self.app.conf.result_backend_always_retry, prev = True, self.app.conf.result_backend_always_retry try: x = ElasticsearchBackend(app=self.app) task_id = str(sentinel.task_id) encoded_task_id = bytes_to_str(x.get_key_for_task(task_id)) result = str(sentinel.result) sleep_mock = Mock() x._sleep = sleep_mock x._server = Mock() x._server.index.side_effect = [ exceptions.ConflictError(409, "concurrent update", {}), { 'result': 'created' } ] x._server.get.side_effect = [ { 'found': True, '_source': { 'result': _RESULT_RETRY }, '_seq_no': 2, '_primary_term': 1, }, { '_index': 'celery', '_type': '_doc', '_id': 'toto', 'found': False }, ] result_meta = x._get_result_meta(result, states.SUCCESS, None, None) result_meta['task_id'] = bytes_to_str(task_id) expected_result = x.encode(result_meta) x.store_result(task_id, result, states.SUCCESS) x._server.index.assert_has_calls([ call(id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' }, params={'op_type': 'create'}), call(id=encoded_task_id, index=x.index, doc_type=x.doc_type, body={ 'result': expected_result, '@timestamp': expected_dt.isoformat()[:-3] + 'Z' }, params={'op_type': 'create'}), ]) x._server.update.assert_not_called() sleep_mock.assert_not_called() finally: self.app.conf.result_backend_always_retry = prev