async def test_mount_snapshot_with_renames(es):
es.snapshot.get.return_value = as_future({
"responses": [{
"repository":
"eventdata",
"snapshots": [{
"snapshot":
"eventdata-snapshot",
"uuid":
"mWJnRABaSh-gdHF3-pexbw",
"indices": [
"elasticlogs-2018-05-03", "elasticlogs-2018-05-04",
"elasticlogs-2018-06-05"
]
}]
}]
})
# one call for each matching index
es.transport.perform_request.side_effect = [
as_future(),
as_future(),
]
params = {
"repository": "eventdata",
"snapshot": "eventdata-snapshot",
"index_pattern": "elasticlogs-2018-05-*",
"rename_pattern": "elasticlogs-(.*)",
"rename_replacement": "renamed-logs-\\1"
}
runner = MountSearchableSnapshotRunner()
await runner(es, params=params)
es.snapshot.get.assert_called_once_with("eventdata", "eventdata-snapshot")
es.transport.perform_request.assert_has_calls([
mock.call(method="POST",
url="/_snapshot/eventdata/eventdata-snapshot/_mount",
body={
"index": "elasticlogs-2018-05-03",
"renamed_index": "renamed-logs-2018-05-03"
},
params=None),
mock.call(method="POST",
url="/_snapshot/eventdata/eventdata-snapshot/_mount",
body={
"index": "elasticlogs-2018-05-04",
"renamed_index": "renamed-logs-2018-05-04"
},
params=None)
])
async def test_mount_snapshot_frozen(es):
es.snapshot.get.return_value = as_future({
"responses": [{
"repository":
"eventdata",
"snapshots": [{
"snapshot":
"eventdata-snapshot",
"uuid":
"mWJnRABaSh-gdHF3-pexbw",
"indices": [
"elasticlogs-2018-05-03", "elasticlogs-2018-05-04",
"elasticlogs-2018-05-05"
]
}]
}]
})
# one call for each index
es.transport.perform_request.side_effect = [
as_future(),
as_future(),
as_future(),
]
params = {
"repository": "eventdata",
"snapshot": "eventdata-snapshot",
"query_params": {
"storage": "shared_cache"
}
}
runner = MountSearchableSnapshotRunner()
await runner(es, params=params)
es.snapshot.get.assert_called_once_with("eventdata", "eventdata-snapshot")
es.transport.perform_request.assert_has_calls([
mock.call(method="POST",
url="/_snapshot/eventdata/eventdata-snapshot/_mount",
body={"index": "elasticlogs-2018-05-03"},
params={"storage": "shared_cache"}),
mock.call(method="POST",
url="/_snapshot/eventdata/eventdata-snapshot/_mount",
body={"index": "elasticlogs-2018-05-04"},
params={"storage": "shared_cache"}),
mock.call(method="POST",
url="/_snapshot/eventdata/eventdata-snapshot/_mount",
body={"index": "elasticlogs-2018-05-05"},
params={"storage": "shared_cache"}),
])
async def test_msearch_without_hits(es):
params = {
"body": [{
"index": "elasticlogs-*"
}, {
"query": {
"match_all": {}
},
"from": 0,
"size": 10
}, {
"index": "elasticlogs-*"
}, {
"query": {
"match_all": {}
},
"from": 0,
"size": 10
}],
"meta_data": {
"debug": True
}
}
es.msearch.return_value = as_future({
"responses": [{
"took": 0,
"timed_out": False,
"hits": {
"total": 0,
"hits": []
},
"status": 200
}, {
"took": 0,
"timed_out": False,
"hits": {
"total": 0,
"hits": []
},
"status": 200
}]
})
response = await kibana(es, params=params)
assert response == {
"debug": True,
"success": True,
"error-count": 0,
"hits": 0,
"took": 0,
"weight": 1,
"unit": "ops",
"visualisation_count": 2,
}
def __init__(self, *args, **kwargs):
StaticClientFactory.SYNC_PATCHER = mock.patch(
"elasticsearch.Elasticsearch")
self.es = StaticClientFactory.SYNC_PATCHER.start()
self.es.indices.stats.return_value = {"mocked": True}
self.es.info.return_value = {
"cluster_name": "elasticsearch",
"version": {
"number": "7.3.0",
"build_flavor": "oss",
"build_type": "tar",
"build_hash": "de777fa",
"build_date": "2019-07-24T18:30:11.767338Z",
"build_snapshot": False,
"lucene_version": "8.1.0",
"minimum_wire_compatibility_version": "6.8.0",
"minimum_index_compatibility_version": "6.0.0-beta1"
}
}
StaticClientFactory.ASYNC_PATCHER = mock.patch(
"elasticsearch.Elasticsearch")
self.es_async = StaticClientFactory.ASYNC_PATCHER.start()
# we want to simulate that the request took 10 seconds. Internally this is measured using `time#perf_counter`
# and the code relies that measurements are taken consistently with `time#perf_counter` because in some places
# we take a value and will subtract other measurements (e.g. in the main loop in AsyncExecutor we subtract
# `total_start` from `stop`.
#
# On some systems (MacOS), `time#perf_counter` starts at zero when the process is started but on others (Linux),
# `time#perf_counter` starts when the OS is started. Thus we need to ensure that this value here is roughly
# consistent across all systems by using the current value of `time#perf_counter` as basis.
start = time.perf_counter()
self.es_async.init_request_context.return_value = {
"request_start": start,
"request_end": start + 10
}
bulk_response = {"errors": False, "took": 5}
# bulk responses are raw strings
self.es_async.bulk.return_value = as_future(
io.StringIO(json.dumps(bulk_response)))
self.es_async.transport.close.return_value = as_future()
async def test_msearch_with_hits_as_number(es):
params = {
"body": [{
"index": "elasticlogs-*"
}, {
"query": {
"match_all": {}
},
"from": 0,
"size": 10
}, {
"index": "elasticlogs-*"
}, {
"query": {
"match_all": {}
},
"from": 0,
"size": 10
}],
"meta_data": {
"debug": True
}
}
es.msearch.return_value = as_future({
"responses": [{
"took": 5,
"timed_out": False,
"hits": {
"total":
1,
"hits": [{
"_index": "my-docs",
"_type": "_doc",
"_id": "1",
"_score": 1,
"_source": {
"title": "Hello"
}
}]
},
"status": 200
}, {
"took": 7,
"timed_out": False,
"hits": {
"total":
2,
"hits": [{
"_index": "my-other-docs",
"_type": "_doc",
"_id": "1",
"_score": 1,
"_source": {
"title": "Hello"
}
}, {
"_index": "my-other-docs",
"_type": "_doc",
"_id": "2",
"_score": 1,
"_source": {
"title": "World"
}
}]
},
"status": 200
}]
})
response = await kibana(es, params=params)
assert response == {
"debug": True,
"success": True,
"error-count": 0,
"hits": 3,
"took": 7,
"weight": 1,
"unit": "ops",
"visualisation_count": 2,
}
async def test_msearch_with_error(es):
params = {
"body": [
{
"index": "elasticlogs-*"
},
{
"query": {
"match_all": {}
},
"from": 0,
"size": 10
},
],
"meta_data": {
"debug": True
}
}
es.msearch.return_value = as_future({
"took":
80000,
"responses": [{
"error": {
"root_cause": [{
"type": "i_o_exception",
"reason": "failed to read data from cache"
}],
"type":
"search_phase_execution_exception",
"reason":
"all shards failed",
"phase":
"fetch",
"grouped":
True,
"failed_shards": [{
"shard": 0,
"index": "logs-2020-01-01",
"node": "Xl9IRaJ6RI-YQRZuBNsbTg",
"reason": {
"type": "i_o_exception",
"reason": "failed to read data from cache"
}
}],
"caused_by": {
"type": "i_o_exception",
"reason": "failed to read data from cache",
}
},
"status": 503
}]
})
response = await kibana(es, params=params)
assert response == {
"debug": True,
"hits": 0,
"took": 80000,
"weight": 1,
"unit": "ops",
"visualisation_count": 1,
"success": False,
"error-count": 1,
"error-type": "kibana",
"error-description": "HTTP status: 503, message: all shards failed"
}
