def test_bulk_index_success_without_metadata(self, es): es.bulk.return_value = { "errors": False } bulk = runner.BulkIndex() bulk_params = { "body": [ "index_line", "index_line", "index_line" ], "action_metadata_present": False, "bulk-size": 3, "index": "test-index", "type": "test-type" } result = bulk(es, bulk_params) self.assertEqual("test-index", result["index"]) self.assertEqual(3, result["weight"]) self.assertEqual(3, result["bulk-size"]) self.assertEqual("docs", result["unit"]) self.assertEqual(True, result["success"]) self.assertEqual(0, result["error-count"]) es.bulk.assert_called_with(body=bulk_params["body"], index="test-index", doc_type="test-type", params={})
def test_bulk_index_success_with_metadata(self, es): es.bulk.return_value = {"errors": False} bulk = runner.BulkIndex() bulk_params = { "body": [ "action_meta_data", "index_line", "action_meta_data", "index_line", "action_meta_data", "index_line" ], "action_metadata_present": True, "bulk-size": 3 } result = bulk(es, bulk_params) self.assertIsNone(result["index"]) self.assertEqual(3, result["weight"]) self.assertEqual(3, result["bulk-size"]) self.assertEqual("docs", result["unit"]) self.assertEqual(True, result["success"]) self.assertEqual(0, result["error-count"]) self.assertFalse("error-type" in result) es.bulk.assert_called_with(body=bulk_params["body"], params={})
def test_bulk_index_error(self, es): es.bulk.return_value = { "errors": True, "items": [ { "index": { "status": 201, "_shards": { "total": 2, "successful": 1, "failed": 0 } } }, { "index": { "status": 500, "_shards": { "total": 2, "successful": 0, "failed": 2 } } }, { "index": { "status": 404, "_shards": { "total": 2, "successful": 0, "failed": 2 } } }, ] } bulk = runner.BulkIndex() bulk_params = { "body": [ "action_meta_data", "index_line", "action_meta_data", "index_line", "action_meta_data", "index_line" ], "action_metadata_present": True } result = bulk(es, bulk_params) self.assertEqual(3, result["weight"]) self.assertEqual(3, result["bulk-size"]) self.assertEqual("docs", result["unit"]) self.assertEqual(False, result["success"]) self.assertEqual(2, result["error-count"]) es.bulk.assert_called_with(body=bulk_params["body"], params={})
def test_bulk_index_missing_params(self, es): es.bulk.return_value = {"errors": False} bulk = runner.BulkIndex() bulk_params = { "body": [ "action_meta_data", "index_line", "action_meta_data", "index_line", "action_meta_data", "index_line" ] } with self.assertRaises(exceptions.DataError) as ctx: bulk(es, bulk_params) self.assertEqual( "Parameter source for operation 'bulk-index' did not provide the mandatory parameter 'action_metadata_present'. " "Please add it to your parameter source.", ctx.exception.args[0])
def test_mixed_bulk_with_detailed_stats(self, es): es.bulk.return_value = { "took": 30, "errors": True, "items": [{ "index": { "_index": "test", "_type": "type1", "_id": "1", "_version": 1, "result": "created", "_shards": { "total": 2, "successful": 1, "failed": 0 }, "created": True, "status": 201, "_seq_no": 0 } }, { "update": { "_index": "test", "_type": "type1", "_id": "2", "_version": 2, "result": "updated", "_shards": { "total": 2, "successful": 1, "failed": 0 }, "status": 200, "_seq_no": 1 } }, { "index": { "_index": "test", "_type": "type1", "_id": "3", "_version": 1, "result": "noop", "_shards": { "total": 2, "successful": 0, "failed": 2 }, "created": False, "status": 500, "_seq_no": -2 } }, { "index": { "_index": "test", "_type": "type1", "_id": "4", "_version": 1, "result": "noop", "_shards": { "total": 2, "successful": 1, "failed": 1 }, "created": False, "status": 500, "_seq_no": -2 } }, { "index": { "_index": "test", "_type": "type1", "_id": "5", "_version": 1, "result": "created", "_shards": { "total": 2, "successful": 1, "failed": 0 }, "created": True, "status": 201, "_seq_no": 4 } }, { "update": { "_index": "test", "_type": "type1", "_id": "6", "_version": 2, "result": "noop", "_shards": { "total": 2, "successful": 0, "failed": 2 }, "status": 404, "_seq_no": 5 } }] } bulk = runner.BulkIndex() bulk_params = { "body": [ "action_meta_data", "index_line", "action_meta_data", "update_line", "action_meta_data", "index_line", "action_meta_data", "index_line", "action_meta_data", "index_line", "action_meta_data", "update_line" ], "action_metadata_present": True, "detailed-results": True } result = bulk(es, bulk_params) self.assertEqual(6, result["weight"]) self.assertEqual(6, result["bulk-size"]) self.assertEqual("docs", result["unit"]) self.assertEqual(False, result["success"]) self.assertEqual(3, result["error-count"]) self.assertEqual( { "index": { "item-count": 4, "created": 2, "noop": 2 }, "update": { "item-count": 2, "updated": 1, "noop": 1 } }, result["ops"]) self.assertEqual([{ "item-count": 3, "shards": { "total": 2, "successful": 1, "failed": 0 } }, { "item-count": 2, "shards": { "total": 2, "successful": 0, "failed": 2 } }, { "item-count": 1, "shards": { "total": 2, "successful": 1, "failed": 1 } }], result["shards_histogram"]) es.bulk.assert_called_with(body=bulk_params["body"], params={})
import pytest from esrally.driver import runner bulk_index = runner.BulkIndex() class ElasticsearchMock: def __init__(self, bulk_size): self.no_errors = {"took": 500, "errors": False, "items": []} for idx in range(0, bulk_size): self.no_errors["items"].append({ "index": { "_index": "test", "_type": "type1", "_id": str(idx), "_version": 1, "result": "created", "_shards": { "total": 2, "successful": 1, "failed": 0 }, "created": True, "status": 201, "_seq_no": 0 } }) def bulk(self, body=None, index=None, doc_type=None, params=None): return self.no_errors
def test_mixed_bulk_with_detailed_stats(self, es): es.bulk.return_value = { "took": 30, "errors": True, "items": [{ "index": { "_index": "test", "_type": "type1", "_id": "1", "_version": 1, "result": "created", "_shards": { "total": 2, "successful": 1, "failed": 0 }, "created": True, "status": 201, "_seq_no": 0 } }, { "update": { "_index": "test", "_type": "type1", "_id": "2", "_version": 2, "result": "updated", "_shards": { "total": 2, "successful": 1, "failed": 0 }, "status": 200, "_seq_no": 1 } }, { "index": { "_index": "test", "_type": "type1", "_id": "3", "_version": 1, "result": "noop", "_shards": { "total": 2, "successful": 0, "failed": 2 }, "created": False, "status": 500, "_seq_no": -2 } }, { "index": { "_index": "test", "_type": "type1", "_id": "4", "_version": 1, "result": "noop", "_shards": { "total": 2, "successful": 1, "failed": 1 }, "created": False, "status": 500, "_seq_no": -2 } }, { "index": { "_index": "test", "_type": "type1", "_id": "5", "_version": 1, "result": "created", "_shards": { "total": 2, "successful": 1, "failed": 0 }, "created": True, "status": 201, "_seq_no": 4 } }, { "update": { "_index": "test", "_type": "type1", "_id": "6", "_version": 2, "result": "noop", "_shards": { "total": 2, "successful": 0, "failed": 2 }, "status": 404, "_seq_no": 5 } }] } bulk = runner.BulkIndex() bulk_params = { "body": [ '{ "index" : { "_index" : "test", "_type" : "type1" } }', '{"location" : [-0.1485188, 51.5250666]}', '{ "update" : { "_index" : "test", "_type" : "type1", "_id: "2" } }', '{"location" : [-0.1479949, 51.5252071]}', '{ "index" : { "_index" : "test", "_type" : "type1" } }', '{"location" : [-0.1458559, 51.5289059]}', '{ "index" : { "_index" : "test", "_type" : "type1" } }', '{"location" : [-0.1498551, 51.5282564]}', '{ "index" : { "_index" : "test", "_type" : "type1" } }', '{"location" : [-0.1487043, 51.5254843]}', '{ "update" : { "_index" : "test", "_type" : "type1", "_id: "3" } }', '{"location" : [-0.1533367, 51.5261779]}' ], "action_metadata_present": True, "bulk-size": 6, "detailed-results": True, "index": "test" } result = bulk(es, bulk_params) self.assertEqual("test", result["index"]) self.assertEqual(6, result["weight"]) self.assertEqual(6, result["bulk-size"]) self.assertEqual("docs", result["unit"]) self.assertEqual(False, result["success"]) self.assertEqual(3, result["error-count"]) self.assertEqual("bulk", result["error-type"]) self.assertEqual( { "index": { "item-count": 4, "created": 2, "noop": 2 }, "update": { "item-count": 2, "updated": 1, "noop": 1 } }, result["ops"]) self.assertEqual([{ "item-count": 3, "shards": { "total": 2, "successful": 1, "failed": 0 } }, { "item-count": 2, "shards": { "total": 2, "successful": 0, "failed": 2 } }, { "item-count": 1, "shards": { "total": 2, "successful": 1, "failed": 1 } }], result["shards_histogram"]) self.assertEqual(582, result["bulk-request-size-bytes"]) self.assertEqual(234, result["total-document-size-bytes"]) es.bulk.assert_called_with(body=bulk_params["body"], params={})