def test_bulk_index_success_without_metadata(self, es):
es.bulk.return_value = {
"errors": False
}
bulk = runner.BulkIndex()
bulk_params = {
"body": [
"index_line",
"index_line",
"index_line"
],
"action_metadata_present": False,
"bulk-size": 3,
"index": "test-index",
"type": "test-type"
}
result = bulk(es, bulk_params)
self.assertEqual("test-index", result["index"])
self.assertEqual(3, result["weight"])
self.assertEqual(3, result["bulk-size"])
self.assertEqual("docs", result["unit"])
self.assertEqual(True, result["success"])
self.assertEqual(0, result["error-count"])
es.bulk.assert_called_with(body=bulk_params["body"], index="test-index", doc_type="test-type", params={})
def test_bulk_index_success_with_metadata(self, es):
es.bulk.return_value = {"errors": False}
bulk = runner.BulkIndex()
bulk_params = {
"body": [
"action_meta_data", "index_line", "action_meta_data",
"index_line", "action_meta_data", "index_line"
],
"action_metadata_present":
True,
"bulk-size":
3
}
result = bulk(es, bulk_params)
self.assertIsNone(result["index"])
self.assertEqual(3, result["weight"])
self.assertEqual(3, result["bulk-size"])
self.assertEqual("docs", result["unit"])
self.assertEqual(True, result["success"])
self.assertEqual(0, result["error-count"])
self.assertFalse("error-type" in result)
es.bulk.assert_called_with(body=bulk_params["body"], params={})
def test_bulk_index_error(self, es):
es.bulk.return_value = {
"errors":
True,
"items": [
{
"index": {
"status": 201,
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
}
}
},
{
"index": {
"status": 500,
"_shards": {
"total": 2,
"successful": 0,
"failed": 2
}
}
},
{
"index": {
"status": 404,
"_shards": {
"total": 2,
"successful": 0,
"failed": 2
}
}
},
]
}
bulk = runner.BulkIndex()
bulk_params = {
"body": [
"action_meta_data", "index_line", "action_meta_data",
"index_line", "action_meta_data", "index_line"
],
"action_metadata_present":
True
}
result = bulk(es, bulk_params)
self.assertEqual(3, result["weight"])
self.assertEqual(3, result["bulk-size"])
self.assertEqual("docs", result["unit"])
self.assertEqual(False, result["success"])
self.assertEqual(2, result["error-count"])
es.bulk.assert_called_with(body=bulk_params["body"], params={})
def test_bulk_index_missing_params(self, es):
es.bulk.return_value = {"errors": False}
bulk = runner.BulkIndex()
bulk_params = {
"body": [
"action_meta_data", "index_line", "action_meta_data",
"index_line", "action_meta_data", "index_line"
]
}
with self.assertRaises(exceptions.DataError) as ctx:
bulk(es, bulk_params)
self.assertEqual(
"Parameter source for operation 'bulk-index' did not provide the mandatory parameter 'action_metadata_present'. "
"Please add it to your parameter source.", ctx.exception.args[0])
def test_mixed_bulk_with_detailed_stats(self, es):
es.bulk.return_value = {
"took":
30,
"errors":
True,
"items": [{
"index": {
"_index": "test",
"_type": "type1",
"_id": "1",
"_version": 1,
"result": "created",
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
},
"created": True,
"status": 201,
"_seq_no": 0
}
}, {
"update": {
"_index": "test",
"_type": "type1",
"_id": "2",
"_version": 2,
"result": "updated",
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
},
"status": 200,
"_seq_no": 1
}
}, {
"index": {
"_index": "test",
"_type": "type1",
"_id": "3",
"_version": 1,
"result": "noop",
"_shards": {
"total": 2,
"successful": 0,
"failed": 2
},
"created": False,
"status": 500,
"_seq_no": -2
}
}, {
"index": {
"_index": "test",
"_type": "type1",
"_id": "4",
"_version": 1,
"result": "noop",
"_shards": {
"total": 2,
"successful": 1,
"failed": 1
},
"created": False,
"status": 500,
"_seq_no": -2
}
}, {
"index": {
"_index": "test",
"_type": "type1",
"_id": "5",
"_version": 1,
"result": "created",
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
},
"created": True,
"status": 201,
"_seq_no": 4
}
}, {
"update": {
"_index": "test",
"_type": "type1",
"_id": "6",
"_version": 2,
"result": "noop",
"_shards": {
"total": 2,
"successful": 0,
"failed": 2
},
"status": 404,
"_seq_no": 5
}
}]
}
bulk = runner.BulkIndex()
bulk_params = {
"body": [
"action_meta_data", "index_line", "action_meta_data",
"update_line", "action_meta_data", "index_line",
"action_meta_data", "index_line", "action_meta_data",
"index_line", "action_meta_data", "update_line"
],
"action_metadata_present":
True,
"detailed-results":
True
}
result = bulk(es, bulk_params)
self.assertEqual(6, result["weight"])
self.assertEqual(6, result["bulk-size"])
self.assertEqual("docs", result["unit"])
self.assertEqual(False, result["success"])
self.assertEqual(3, result["error-count"])
self.assertEqual(
{
"index": {
"item-count": 4,
"created": 2,
"noop": 2
},
"update": {
"item-count": 2,
"updated": 1,
"noop": 1
}
}, result["ops"])
self.assertEqual([{
"item-count": 3,
"shards": {
"total": 2,
"successful": 1,
"failed": 0
}
}, {
"item-count": 2,
"shards": {
"total": 2,
"successful": 0,
"failed": 2
}
}, {
"item-count": 1,
"shards": {
"total": 2,
"successful": 1,
"failed": 1
}
}], result["shards_histogram"])
es.bulk.assert_called_with(body=bulk_params["body"], params={})
import pytest
from esrally.driver import runner
bulk_index = runner.BulkIndex()
class ElasticsearchMock:
def __init__(self, bulk_size):
self.no_errors = {"took": 500, "errors": False, "items": []}
for idx in range(0, bulk_size):
self.no_errors["items"].append({
"index": {
"_index": "test",
"_type": "type1",
"_id": str(idx),
"_version": 1,
"result": "created",
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
},
"created": True,
"status": 201,
"_seq_no": 0
}
})
def bulk(self, body=None, index=None, doc_type=None, params=None):
return self.no_errors
def test_mixed_bulk_with_detailed_stats(self, es):
es.bulk.return_value = {
"took":
30,
"errors":
True,
"items": [{
"index": {
"_index": "test",
"_type": "type1",
"_id": "1",
"_version": 1,
"result": "created",
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
},
"created": True,
"status": 201,
"_seq_no": 0
}
}, {
"update": {
"_index": "test",
"_type": "type1",
"_id": "2",
"_version": 2,
"result": "updated",
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
},
"status": 200,
"_seq_no": 1
}
}, {
"index": {
"_index": "test",
"_type": "type1",
"_id": "3",
"_version": 1,
"result": "noop",
"_shards": {
"total": 2,
"successful": 0,
"failed": 2
},
"created": False,
"status": 500,
"_seq_no": -2
}
}, {
"index": {
"_index": "test",
"_type": "type1",
"_id": "4",
"_version": 1,
"result": "noop",
"_shards": {
"total": 2,
"successful": 1,
"failed": 1
},
"created": False,
"status": 500,
"_seq_no": -2
}
}, {
"index": {
"_index": "test",
"_type": "type1",
"_id": "5",
"_version": 1,
"result": "created",
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
},
"created": True,
"status": 201,
"_seq_no": 4
}
}, {
"update": {
"_index": "test",
"_type": "type1",
"_id": "6",
"_version": 2,
"result": "noop",
"_shards": {
"total": 2,
"successful": 0,
"failed": 2
},
"status": 404,
"_seq_no": 5
}
}]
}
bulk = runner.BulkIndex()
bulk_params = {
"body": [
'{ "index" : { "_index" : "test", "_type" : "type1" } }',
'{"location" : [-0.1485188, 51.5250666]}',
'{ "update" : { "_index" : "test", "_type" : "type1", "_id: "2" } }',
'{"location" : [-0.1479949, 51.5252071]}',
'{ "index" : { "_index" : "test", "_type" : "type1" } }',
'{"location" : [-0.1458559, 51.5289059]}',
'{ "index" : { "_index" : "test", "_type" : "type1" } }',
'{"location" : [-0.1498551, 51.5282564]}',
'{ "index" : { "_index" : "test", "_type" : "type1" } }',
'{"location" : [-0.1487043, 51.5254843]}',
'{ "update" : { "_index" : "test", "_type" : "type1", "_id: "3" } }',
'{"location" : [-0.1533367, 51.5261779]}'
],
"action_metadata_present":
True,
"bulk-size":
6,
"detailed-results":
True,
"index":
"test"
}
result = bulk(es, bulk_params)
self.assertEqual("test", result["index"])
self.assertEqual(6, result["weight"])
self.assertEqual(6, result["bulk-size"])
self.assertEqual("docs", result["unit"])
self.assertEqual(False, result["success"])
self.assertEqual(3, result["error-count"])
self.assertEqual("bulk", result["error-type"])
self.assertEqual(
{
"index": {
"item-count": 4,
"created": 2,
"noop": 2
},
"update": {
"item-count": 2,
"updated": 1,
"noop": 1
}
}, result["ops"])
self.assertEqual([{
"item-count": 3,
"shards": {
"total": 2,
"successful": 1,
"failed": 0
}
}, {
"item-count": 2,
"shards": {
"total": 2,
"successful": 0,
"failed": 2
}
}, {
"item-count": 1,
"shards": {
"total": 2,
"successful": 1,
"failed": 1
}
}], result["shards_histogram"])
self.assertEqual(582, result["bulk-request-size-bytes"])
self.assertEqual(234, result["total-document-size-bytes"])
es.bulk.assert_called_with(body=bulk_params["body"], params={})
