def test_get_with_indices(self, mocked_es):
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
# Test default indices.
self.api.post(
query='{}'
)
mocked_connection.search.assert_called_with(
body={},
index=[self.api.config.elasticsearch.elasticsearch_index],
doc_type=self.api.config.elasticsearch.elasticsearch_doctype
)
# Test all indices.
self.api.post(
query='{}',
indices=['ALL']
)
mocked_connection.search.assert_called_with(
body={}
)
# Test forcing indices.
self.api.post(
query='{}',
indices=['socorro_201801', 'socorro_200047', 'not_an_index']
)
mocked_connection.search.assert_called_with(
body={},
index=['socorro_201801', 'socorro_200047', 'not_an_index'],
doc_type=self.api.config.elasticsearch.elasticsearch_doctype
)
# Test default indices with an index schema based on dates.
index_schema = 'socorro_%Y%W'
config = self.get_base_config(es_index=index_schema)
api = Query(config=config)
now = datetimeutil.utc_now()
last_week = now - datetime.timedelta(days=7)
indices = api.generate_list_of_indexes(last_week, now)
api.post(
query='{}'
)
mocked_connection.search.assert_called_with(
body={},
index=indices,
doc_type=api.config.elasticsearch.elasticsearch_doctype
)
def test_get_with_indices(self, mocked_es):
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
# Test indices with dates (the test configuration includes dates).
self.api.get(query={"query": {}})
now = utc_now()
last_week = now - datetime.timedelta(days=7)
indices = generate_list_of_indexes(
last_week, now, self.api.context.get_index_template())
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=indices,
doc_type=self.es_context.get_doctype(),
)
# Test all indices.
self.api.get(query={"query": {}}, indices=["ALL"])
mocked_connection.search.assert_called_with(body='{"query": {}}')
# Test forcing indices.
self.api.get(
query={"query": {}},
indices=["socorro_201801", "socorro_200047", "not_an_index"],
)
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=["socorro_201801", "socorro_200047", "not_an_index"],
doc_type=self.es_context.get_doctype(),
)
# Test default indices with an index schema based on dates.
index_schema = "testsocorro"
config = self.get_base_config(cls=Query, es_index=index_schema)
api = Query(config=config)
api.get(query={"query": {}})
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=["testsocorro"],
doc_type=api.context.get_doctype(),
)
def __init__(self, *args, **kwargs):
super(IntegrationTestQuery, self).__init__(*args, **kwargs)
self.api = Query(config=self.config)
class IntegrationTestQuery(ElasticsearchTestCase):
"""Test Query with an elasticsearch database containing fake data. """
def __init__(self, *args, **kwargs):
super(IntegrationTestQuery, self).__init__(*args, **kwargs)
self.api = Query(config=self.config)
def test_get(self):
self.index_crash({'product': 'WaterWolf'})
self.index_crash({'product': 'EarthRaccoon'})
self.refresh_index()
query = {
'query': {
'match_all': {}
}
}
res = self.api.post(query=json.dumps(query))
ok_(res)
ok_('hits' in res)
eq_(res['hits']['total'], 2)
query = {
'query': {
'filtered': {
'query': {
'match_all': {}
},
'filter': {
'term': {
'product': 'earthraccoon'
}
}
}
}
}
res = self.api.post(query=json.dumps(query))
ok_(res)
ok_('hits' in res)
eq_(res['hits']['total'], 1)
@mock.patch('socorro.external.es.connection_context.elasticsearch')
def test_get_with_errors(self, mocked_es):
# Test missing argument.
assert_raises(
BadArgumentError,
self.api.post,
query='hello!',
)
# Test invalid JSON argument.
assert_raises(
MissingArgumentError,
self.api.post,
)
# Test missing index in elasticsearch.
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
mocked_connection.search.side_effect = (
elasticsearch.exceptions.NotFoundError(
404, '[[socorro_201801] missing]'
)
)
assert_raises(
ResourceNotFound,
self.api.post,
query='{}',
)
# Test a generic error response from elasticsearch.
mocked_connection.search.side_effect = (
elasticsearch.exceptions.TransportError('aaa')
)
assert_raises(
DatabaseError,
self.api.post,
query='{}',
)
@mock.patch('socorro.external.es.connection_context.elasticsearch')
def test_get_with_indices(self, mocked_es):
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
# Test default indices.
self.api.post(
query='{}'
)
mocked_connection.search.assert_called_with(
body={},
index=[self.api.config.elasticsearch.elasticsearch_index],
doc_type=self.api.config.elasticsearch.elasticsearch_doctype
)
# Test all indices.
self.api.post(
query='{}',
indices=['ALL']
)
mocked_connection.search.assert_called_with(
body={}
)
# Test forcing indices.
self.api.post(
query='{}',
indices=['socorro_201801', 'socorro_200047', 'not_an_index']
)
mocked_connection.search.assert_called_with(
body={},
index=['socorro_201801', 'socorro_200047', 'not_an_index'],
doc_type=self.api.config.elasticsearch.elasticsearch_doctype
)
# Test default indices with an index schema based on dates.
index_schema = 'socorro_%Y%W'
config = self.get_base_config(es_index=index_schema)
api = Query(config=config)
now = datetimeutil.utc_now()
last_week = now - datetime.timedelta(days=7)
indices = api.generate_list_of_indexes(last_week, now)
api.post(
query='{}'
)
mocked_connection.search.assert_called_with(
body={},
index=indices,
doc_type=api.config.elasticsearch.elasticsearch_doctype
)
class TestIntegrationQuery(ElasticsearchTestCase):
"""Test Query with an elasticsearch database containing fake data."""
def setup_method(self):
super().setup_method()
config = self.get_base_config(cls=Query)
self.api = Query(config=config)
def test_get(self):
datestamp = date_to_string(utc_now())
self.index_crash(processed_crash={
"date_processed": datestamp,
"product": "WaterWolf"
})
self.index_crash(processed_crash={
"date_processed": datestamp,
"product": "EarthRaccoon"
})
self.es_context.refresh()
query = {"query": {"match_all": {}}}
res = self.api.get(query=query)
assert res["hits"]["total"] == 2
query = {
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"term": {
"product": "earthraccoon"
}
},
}
}
}
res = self.api.get(query=query)
assert res["hits"]["total"] == 1
def test_get_with_errors(self):
# Test missing argument.
with pytest.raises(MissingArgumentError):
self.api.get()
with pytest.raises(ResourceNotFound):
query = {"query": {"match_all": {}}}
self.api.get(query=query, indices=["not_an_index"])
with pytest.raises(DatabaseError):
self.api.get(query={"query": {}})
@mock.patch("socorro.external.es.connection_context.elasticsearch")
def test_get_with_indices(self, mocked_es):
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
# Test indices with dates (the test configuration includes dates).
self.api.get(query={"query": {}})
now = utc_now()
last_week = now - datetime.timedelta(days=7)
indices = generate_list_of_indexes(
last_week, now, self.api.context.get_index_template())
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=indices,
doc_type=self.es_context.get_doctype(),
)
# Test all indices.
self.api.get(query={"query": {}}, indices=["ALL"])
mocked_connection.search.assert_called_with(body='{"query": {}}')
# Test forcing indices.
self.api.get(
query={"query": {}},
indices=["socorro_201801", "socorro_200047", "not_an_index"],
)
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=["socorro_201801", "socorro_200047", "not_an_index"],
doc_type=self.es_context.get_doctype(),
)
# Test default indices with an index schema based on dates.
index_schema = "testsocorro"
config = self.get_base_config(cls=Query, es_index=index_schema)
api = Query(config=config)
api.get(query={"query": {}})
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=["testsocorro"],
doc_type=api.context.get_doctype(),
)
def setup_method(self):
super().setup_method()
config = self.get_base_config(cls=Query)
self.api = Query(config=config)
class TestIntegrationQuery(ElasticsearchTestCase):
"""Test Query with an elasticsearch database containing fake data."""
def setup_method(self, method):
super().setup_method(method)
config = self.get_base_config(cls=Query)
self.api = Query(config=config)
def test_get(self):
self.index_crash({"product": "WaterWolf"})
self.index_crash({"product": "EarthRaccoon"})
self.es_context.refresh()
query = {"query": {"match_all": {}}}
res = self.api.get(query=query)
assert res
assert "hits" in res
assert res["hits"]["total"] == 2
query = {
"query": {
"filtered": {
"query": {"match_all": {}},
"filter": {"term": {"product": "earthraccoon"}},
}
}
}
res = self.api.get(query=query)
assert res
assert "hits" in res
assert res["hits"]["total"] == 1
@mock.patch("socorro.external.es.connection_context.elasticsearch")
def test_get_with_errors(self, mocked_es):
# Test missing argument.
with pytest.raises(MissingArgumentError):
self.api.get()
# Test missing index in elasticsearch.
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
mocked_connection.search.side_effect = elasticsearch.exceptions.NotFoundError(
404, "[[socorro_201801] missing]"
)
with pytest.raises(ResourceNotFound):
self.api.get(query={"query": {}})
# Test a generic error response from elasticsearch.
mocked_connection.search.side_effect = elasticsearch.exceptions.TransportError(
"aaa"
)
with pytest.raises(DatabaseError):
self.api.get(query={"query": {}})
@mock.patch("socorro.external.es.connection_context.elasticsearch")
def test_get_with_indices(self, mocked_es):
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
# Test default indices.
self.api.get(query={"query": {}})
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=[self.es_context.get_index_template()],
doc_type=self.es_context.get_doctype(),
)
# Test all indices.
self.api.get(query={"query": {}}, indices=["ALL"])
mocked_connection.search.assert_called_with(body='{"query": {}}')
# Test forcing indices.
self.api.get(
query={"query": {}},
indices=["socorro_201801", "socorro_200047", "not_an_index"],
)
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=["socorro_201801", "socorro_200047", "not_an_index"],
doc_type=self.es_context.get_doctype(),
)
# Test default indices with an index schema based on dates.
index_schema = "socorro_%Y%W"
config = self.get_base_config(cls=Query, es_index=index_schema)
api = Query(config=config)
now = datetimeutil.utc_now()
last_week = now - datetime.timedelta(days=7)
indices = generate_list_of_indexes(
last_week, now, api.context.get_index_template()
)
api.get(query={"query": {}})
mocked_connection.search.assert_called_with(
body='{"query": {}}', index=indices, doc_type=api.context.get_doctype()
)
class TestIntegrationQuery(ElasticsearchTestCase):
"""Test Query with an elasticsearch database containing fake data."""
def setup_method(self, method):
super().setup_method(method)
config = self.get_base_config(cls=Query)
self.api = Query(config=config)
def test_get(self):
self.index_crash({'product': 'WaterWolf'})
self.index_crash({'product': 'EarthRaccoon'})
self.refresh_index()
query = {'query': {'match_all': {}}}
res = self.api.get(query=query)
assert res
assert 'hits' in res
assert res['hits']['total'] == 2
query = {
'query': {
'filtered': {
'query': {
'match_all': {}
},
'filter': {
'term': {
'product': 'earthraccoon'
}
}
}
}
}
res = self.api.get(query=query)
assert res
assert 'hits' in res
assert res['hits']['total'] == 1
@mock.patch('socorro.external.es.connection_context.elasticsearch')
def test_get_with_errors(self, mocked_es):
# Test missing argument.
with pytest.raises(MissingArgumentError):
self.api.get()
# Test missing index in elasticsearch.
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
mocked_connection.search.side_effect = (
elasticsearch.exceptions.NotFoundError(
404, '[[socorro_201801] missing]'))
with pytest.raises(ResourceNotFound):
self.api.get(query={'query': {}})
# Test a generic error response from elasticsearch.
mocked_connection.search.side_effect = (
elasticsearch.exceptions.TransportError('aaa'))
with pytest.raises(DatabaseError):
self.api.get(query={'query': {}})
@mock.patch('socorro.external.es.connection_context.elasticsearch')
def test_get_with_indices(self, mocked_es):
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
# Test default indices.
self.api.get(query={'query': {}})
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=[self.es_context.get_index_template()],
doc_type=self.es_context.get_doctype())
# Test all indices.
self.api.get(query={'query': {}}, indices=['ALL'])
mocked_connection.search.assert_called_with(body='{"query": {}}')
# Test forcing indices.
self.api.get(
query={'query': {}},
indices=['socorro_201801', 'socorro_200047', 'not_an_index'])
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=['socorro_201801', 'socorro_200047', 'not_an_index'],
doc_type=self.es_context.get_doctype())
# Test default indices with an index schema based on dates.
index_schema = 'socorro_%Y%W'
config = self.get_base_config(cls=Query, es_index=index_schema)
api = Query(config=config)
now = datetimeutil.utc_now()
last_week = now - datetime.timedelta(days=7)
indices = generate_list_of_indexes(last_week, now,
api.context.get_index_template())
api.get(query={'query': {}})
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=indices,
doc_type=api.context.get_doctype())
def setup_method(self, method):
super(IntegrationTestQuery, self).setup_method(method)
self.api = Query(config=self.config)
def setup_method(self, method):
super().setup_method(method)
config = self.get_base_config(cls=Query)
self.api = Query(config=config)
class TestIntegrationQuery(ElasticsearchTestCase):
"""Test Query with an elasticsearch database containing fake data."""
def setup_method(self, method):
super().setup_method(method)
config = self.get_base_config(cls=Query)
self.api = Query(config=config)
def test_get(self):
self.index_crash({'product': 'WaterWolf'})
self.index_crash({'product': 'EarthRaccoon'})
self.refresh_index()
query = {
'query': {
'match_all': {}
}
}
res = self.api.get(query=query)
assert res
assert 'hits' in res
assert res['hits']['total'] == 2
query = {
'query': {
'filtered': {
'query': {
'match_all': {}
},
'filter': {
'term': {
'product': 'earthraccoon'
}
}
}
}
}
res = self.api.get(query=query)
assert res
assert 'hits' in res
assert res['hits']['total'] == 1
@mock.patch('socorro.external.es.connection_context.elasticsearch')
def test_get_with_errors(self, mocked_es):
# Test missing argument.
with pytest.raises(MissingArgumentError):
self.api.get()
# Test missing index in elasticsearch.
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
mocked_connection.search.side_effect = (
elasticsearch.exceptions.NotFoundError(
404, '[[socorro_201801] missing]'
)
)
with pytest.raises(ResourceNotFound):
self.api.get(query={'query': {}})
# Test a generic error response from elasticsearch.
mocked_connection.search.side_effect = (
elasticsearch.exceptions.TransportError('aaa')
)
with pytest.raises(DatabaseError):
self.api.get(query={'query': {}})
@mock.patch('socorro.external.es.connection_context.elasticsearch')
def test_get_with_indices(self, mocked_es):
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
# Test default indices.
self.api.get(
query={'query': {}}
)
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=[self.es_context.get_index_template()],
doc_type=self.es_context.get_doctype()
)
# Test all indices.
self.api.get(
query={'query': {}},
indices=['ALL']
)
mocked_connection.search.assert_called_with(
body='{"query": {}}'
)
# Test forcing indices.
self.api.get(
query={'query': {}},
indices=['socorro_201801', 'socorro_200047', 'not_an_index']
)
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=['socorro_201801', 'socorro_200047', 'not_an_index'],
doc_type=self.es_context.get_doctype()
)
# Test default indices with an index schema based on dates.
index_schema = 'socorro_%Y%W'
config = self.get_base_config(cls=Query, es_index=index_schema)
api = Query(config=config)
now = datetimeutil.utc_now()
last_week = now - datetime.timedelta(days=7)
indices = generate_list_of_indexes(last_week, now, api.context.get_index_template())
api.get(
query={'query': {}}
)
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=indices,
doc_type=api.context.get_doctype()
)