def test_backend_index_corrupted_conflicting_document(
self, base_datetime_mock, es_datetime_mock):
expected_dt = datetime.datetime(2020, 6, 1, 18, 43, 24, 123456, None)
es_datetime_mock.utcnow.return_value = expected_dt
expected_done_dt = datetime.datetime(2020, 6, 1, 18, 45, 34, 654321,
None)
base_datetime_mock.utcnow.return_value = expected_done_dt
# self.app.conf.result_backend_always_retry, prev = True, self.app.conf.result_backend_always_retry
# try:
x = ElasticsearchBackend(app=self.app)
task_id = str(sentinel.task_id)
encoded_task_id = bytes_to_str(x.get_key_for_task(task_id))
result = str(sentinel.result)
sleep_mock = Mock()
x._sleep = sleep_mock
x._server = Mock()
x._server.index.side_effect = [
exceptions.ConflictError(409, "concurrent update", {})
]
x._server.update.side_effect = [{'result': 'updated'}]
x._server.get.return_value = {
'found': True,
'_source': {},
'_seq_no': 2,
'_primary_term': 1,
}
result_meta = x._get_result_meta(result, states.SUCCESS, None, None)
result_meta['task_id'] = bytes_to_str(task_id)
expected_result = x.encode(result_meta)
x.store_result(task_id, result, states.SUCCESS)
x._server.index.assert_called_once_with(
id=encoded_task_id,
index=x.index,
doc_type=x.doc_type,
body={
'result': expected_result,
'@timestamp': expected_dt.isoformat()[:-3] + 'Z'
},
params={'op_type': 'create'})
x._server.update.assert_called_once_with(
id=encoded_task_id,
index=x.index,
doc_type=x.doc_type,
body={
'doc': {
'result': expected_result,
'@timestamp': expected_dt.isoformat()[:-3] + 'Z'
}
},
params={
'if_primary_term': 1,
'if_seq_no': 2
})
sleep_mock.assert_not_called()
def test_backend_concurrent_update(self, base_datetime_mock,
es_datetime_mock):
expected_dt = datetime.datetime(2020, 6, 1, 18, 43, 24, 123456, None)
es_datetime_mock.utcnow.return_value = expected_dt
expected_done_dt = datetime.datetime(2020, 6, 1, 18, 45, 34, 654321,
None)
base_datetime_mock.utcnow.return_value = expected_done_dt
self.app.conf.result_backend_always_retry, prev = True, self.app.conf.result_backend_always_retry
try:
x = ElasticsearchBackend(app=self.app)
task_id = str(sentinel.task_id)
encoded_task_id = bytes_to_str(x.get_key_for_task(task_id))
result = str(sentinel.result)
sleep_mock = Mock()
x._sleep = sleep_mock
x._server = Mock()
x._server.index.side_effect = exceptions.ConflictError(
409, "concurrent update", {})
x._server.get.side_effect = [
{
'found': True,
'_source': {
'result':
"""{"status":"RETRY","result":{"exc_type":"Exception","exc_message":["failed"],"exc_module":"builtins"}}"""
},
'_seq_no': 2,
'_primary_term': 1,
},
{
'found': True,
'_source': {
'result':
"""{"status":"RETRY","result":{"exc_type":"Exception","exc_message":["failed"],"exc_module":"builtins"}}"""
},
'_seq_no': 2,
'_primary_term': 1,
},
{
'found': True,
'_source': {
'result':
"""{"status":"FAILURE","result":{"exc_type":"Exception","exc_message":["failed"],"exc_module":"builtins"}}"""
},
'_seq_no': 3,
'_primary_term': 1,
},
{
'found': True,
'_source': {
'result':
"""{"status":"FAILURE","result":{"exc_type":"Exception","exc_message":["failed"],"exc_module":"builtins"}}"""
},
'_seq_no': 3,
'_primary_term': 1,
},
]
x._server.update.side_effect = [{
'result': 'noop'
}, {
'result': 'updated'
}]
result_meta = x._get_result_meta(result, states.SUCCESS, None,
None)
result_meta['task_id'] = bytes_to_str(task_id)
expected_result = x.encode(result_meta)
x.store_result(task_id, result, states.SUCCESS)
x._server.index.assert_has_calls([
call(id=encoded_task_id,
index=x.index,
doc_type=x.doc_type,
body={
'result': expected_result,
'@timestamp': expected_dt.isoformat()[:-3] + 'Z'
},
params={'op_type': 'create'}),
call(id=encoded_task_id,
index=x.index,
doc_type=x.doc_type,
body={
'result': expected_result,
'@timestamp': expected_dt.isoformat()[:-3] + 'Z'
},
params={'op_type': 'create'}),
])
x._server.update.assert_has_calls([
call(id=encoded_task_id,
index=x.index,
doc_type=x.doc_type,
body={
'doc': {
'result': expected_result,
'@timestamp': expected_dt.isoformat()[:-3] + 'Z'
}
},
params={
'if_seq_no': 2,
'if_primary_term': 1
}),
call(id=encoded_task_id,
index=x.index,
doc_type=x.doc_type,
body={
'doc': {
'result': expected_result,
'@timestamp': expected_dt.isoformat()[:-3] + 'Z'
}
},
params={
'if_seq_no': 3,
'if_primary_term': 1
}),
])
assert sleep_mock.call_count == 1
finally:
self.app.conf.result_backend_always_retry = prev
def test_backend_index_conflicting_document_removed_not_throwing(
self, base_datetime_mock, es_datetime_mock):
expected_dt = datetime.datetime(2020, 6, 1, 18, 43, 24, 123456, None)
es_datetime_mock.utcnow.return_value = expected_dt
expected_done_dt = datetime.datetime(2020, 6, 1, 18, 45, 34, 654321,
None)
base_datetime_mock.utcnow.return_value = expected_done_dt
self.app.conf.result_backend_always_retry, prev = True, self.app.conf.result_backend_always_retry
try:
x = ElasticsearchBackend(app=self.app)
task_id = str(sentinel.task_id)
encoded_task_id = bytes_to_str(x.get_key_for_task(task_id))
result = str(sentinel.result)
sleep_mock = Mock()
x._sleep = sleep_mock
x._server = Mock()
x._server.index.side_effect = [
exceptions.ConflictError(409, "concurrent update", {}), {
'result': 'created'
}
]
x._server.get.side_effect = [
{
'found': True,
'_source': {
'result': _RESULT_RETRY
},
'_seq_no': 2,
'_primary_term': 1,
},
{
'_index': 'celery',
'_type': '_doc',
'_id': 'toto',
'found': False
},
]
result_meta = x._get_result_meta(result, states.SUCCESS, None,
None)
result_meta['task_id'] = bytes_to_str(task_id)
expected_result = x.encode(result_meta)
x.store_result(task_id, result, states.SUCCESS)
x._server.index.assert_has_calls([
call(id=encoded_task_id,
index=x.index,
doc_type=x.doc_type,
body={
'result': expected_result,
'@timestamp': expected_dt.isoformat()[:-3] + 'Z'
},
params={'op_type': 'create'}),
call(id=encoded_task_id,
index=x.index,
doc_type=x.doc_type,
body={
'result': expected_result,
'@timestamp': expected_dt.isoformat()[:-3] + 'Z'
},
params={'op_type': 'create'}),
])
x._server.update.assert_not_called()
sleep_mock.assert_not_called()
finally:
self.app.conf.result_backend_always_retry = prev
