def test_es_indices(self, m_conn):
"""Indices tests.
To avoid Elasticsearch calls, we mock out the get_connection call, then
set up additional mocks for the resulting ES connection.
"""
m_es = mock.Mock(Elasticsearch, name='es')
m_indices = mock.MagicMock(IndicesClient, name='indices')
m_es.indices = m_indices
m_es.indices.status.return_value = {
'indices': {
'index1': 'value1',
'not_index1': 'value3'
}
}
m_conn.return_value = m_es
# test with no prefix provided
self.assertEqual(es_indices(conn=es_conn()), "_all")
# test with no params
self.assertEqual(es_indices(), "_all")
# test with no conn
result = es_indices(prefix='index')
self.assertTrue(m_es.indices.status.called)
self.assertIn('index1', result)
self.assertNotIn('not_index1', result)
# test with prefix
result = es_indices('index', es_conn())
self.assertIn('index1', result)
self.assertNotIn('not_index1', result)
def test_es_indices(self, m_conn):
"""Indices tests.
To avoid Elasticsearch calls, we mock out the get_connection call, then
set up additional mocks for the resulting ES connection.
"""
m_es = mock.Mock(Elasticsearch, name="es")
m_indices = mock.MagicMock(IndicesClient, name="indices")
m_es.indices = m_indices
m_es.indices.status.return_value = {"indices": {"index1": "value1", "not_index1": "value3"}}
m_conn.return_value = m_es
# test with no prefix provided
self.assertEqual(es_indices(conn=es_conn()), "_all")
# test with no params
self.assertEqual(es_indices(), "_all")
# test with no conn
result = es_indices(prefix="index")
self.assertTrue(m_es.indices.status.called)
self.assertIn("index1", result)
self.assertNotIn("not_index1", result)
# test with prefix
result = es_indices("index", es_conn())
self.assertIn("index1", result)
self.assertNotIn("not_index1", result)
def test_connection(self, mock_conf, mock_es):
"""Connection tests."""
mock_es.return_value = None
mock_conf.return_value = None
es_conn()
self.assertEqual(mock_es.call_count, 1)
mock_conf.assert_called_with(default=settings.ES_SERVER, sniff_on_start=False, max_retries=1)
mock_conf.reset_mock()
mock_es.reset_mock()
es_conn(server={"hosts": ["abc", "def"]})
self.assertEqual(mock_es.call_count, 1)
mock_conf.assert_called_with(default={"hosts": ["abc", "def"]}, sniff_on_start=False, max_retries=1)
def nova_hypervisors_stats():
"""Get stats from the nova API and add them as Goldstone metrics."""
from goldstone.models import es_conn, daily_index
novaclient = get_nova_client()
response = novaclient.hypervisors.statistics()._info
region = get_keystone_region()
metric_prefix = 'nova.hypervisor.'
now = arrow.utcnow()
conn = es_conn()
es_index = daily_index(METRIC_INDEX_PREFIX)
es_doc_type = METRIC_DOCTYPE
for key, value in response.items():
doc = {
'type': es_doc_type,
'name': metric_prefix + key,
'value': value,
'metric_type': 'gauge',
'@timestamp': now.isoformat(),
'region': region
}
if key in ['disk_available_least', 'free_disk_gb', 'local_gb',
'local_gb_used']:
doc['unit'] = 'GB'
elif key in ['free_ram_mb', 'memory_mb', 'memory_mb_used']:
doc['unit'] = 'MB'
else:
doc['unit'] = 'count'
conn.create(es_index, es_doc_type, doc)
def nova_hypervisors_stats():
"""Get stats from the nova API and add them as Goldstone metrics."""
from goldstone.utils import get_nova_client
from goldstone.models import es_conn, daily_index
novaclient = get_nova_client()['client']
response = \
novaclient.hypervisors.statistics()._info # pylint: disable=W0212
region = get_region_for_nova_client(novaclient)
metric_prefix = 'nova.hypervisor.'
now = arrow.utcnow()
conn = es_conn()
es_index = daily_index(MetricData.INDEX_PREFIX)
es_doc_type = MetricData._doc_type.name # pylint: disable=W0212
for key, value in response.items():
doc = {
'type': es_doc_type,
'name': metric_prefix + key,
'value': value,
'metric_type': 'gauge',
'@timestamp': now.isoformat(),
'region': region
}
if key in ['disk_available_least', 'free_disk_gb', 'local_gb',
'local_gb_used']:
doc['unit'] = 'GB'
elif key in ['free_ram_mb', 'memory_mb', 'memory_mb_used']:
doc['unit'] = 'MB'
else:
doc['unit'] = 'count'
conn.create(es_index, es_doc_type, doc)
def search(cls):
"""Gets a generic Log search object.
See elasticsearch-dsl for parameter information.
"""
return Search(
index=es_indices(cls.INDEX_PREFIX),
doc_type={cls._doc_type.name: cls.from_es},
).sort(cls.SORT).using(es_conn())
def test_connection(self, mock_conf, mock_es):
"""Connection tests."""
mock_es.return_value = None
mock_conf.return_value = None
es_conn()
self.assertEqual(mock_es.call_count, 1)
mock_conf.assert_called_with(default=settings.ES_SERVER,
sniff_on_start=False,
max_retries=1)
mock_conf.reset_mock()
mock_es.reset_mock()
es_conn(server={'hosts': ['abc', 'def']})
self.assertEqual(mock_es.call_count, 1)
mock_conf.assert_called_with(default={'hosts': ['abc', 'def']},
sniff_on_start=False,
max_retries=1)
def get_field_mapping(cls, field):
"""Return a field mapping."""
conn = es_conn()
index = es_indices(cls.INDEX_PREFIX)
return conn.indices.get_field_mapping(field,
index,
cls._doc_type.name,
include_defaults=True,
allow_no_indices=False)
def save(self, using=None, index=None, **kwargs):
"""Posts a record to the database.
See elasticsearch-dsl for parameter information.
"""
if using is None:
using = es_conn()
if index is None:
index = daily_index(self.INDEX_PREFIX)
return super(DailyIndexDocType, self).save(using, index, **kwargs)
def prune_es_indices():
"""Prune ES indices older than the age defined in
settings.PRUNE_OLDER_THAN."""
curation_params = [
{"prefix": "events_", "time_string": "%Y-%m-%d"},
{"prefix": "logstash-", "time_string": "%Y.%m.%d"},
{"prefix": "goldstone-", "time_string": "%Y.%m.%d"},
{"prefix": "goldstone_metrics-", "time_string": "%Y.%m.%d"},
{"prefix": "api_stats-", "time_string": "%Y.%m.%d"},
{"prefix": "internal-", "time_string": "%Y.%m.%d"},
]
client = es_conn()
all_indices = curator.get_indices(client)
deleted_indices = []
working_list = all_indices # we'll whittle this down with filters
for index_set in curation_params:
# filter on our prefix
name_filter = curator.build_filter(
kindOf='prefix', value=index_set['prefix'])
# filter on the datestring
age_filter = curator.build_filter(
kindOf='older_than', time_unit='days',
timestring=index_set['time_string'],
value=settings.PRUNE_OLDER_THAN)
# apply the filters to get the final list of indices to delete
working_list = curator.apply_filter(working_list, **name_filter)
working_list = curator.apply_filter(working_list, **age_filter)
if working_list is not None and len(working_list) > 0:
try:
curator.delete_indices(client, working_list)
deleted_indices = deleted_indices + working_list
except Exception:
logger.exception("curator.delete_indices raised an exception")
working_list = all_indices # reset for the next loop iteration
return deleted_indices
def prune_es_indices():
"""Prune ES indices older than the age defined in
settings.PRUNE_OLDER_THAN."""
curation_params = [
{"prefix": "events_", "time_string": "%Y-%m-%d"},
{"prefix": "logstash-", "time_string": "%Y.%m.%d"},
{"prefix": "goldstone-", "time_string": "%Y.%m.%d"},
{"prefix": "goldstone_metrics-", "time_string": "%Y.%m.%d"},
{"prefix": "api_stats-", "time_string": "%Y.%m.%d"},
{"prefix": "internal-", "time_string": "%Y.%m.%d"},
]
client = es_conn()
all_indices = curator.get_indices(client)
deleted_indices = []
working_list = all_indices # we'll whittle this down with filters
for index_set in curation_params:
# filter on our prefix
name_filter = curator.build_filter(
kindOf='prefix', value=index_set['prefix'])
# filter on the datestring
age_filter = curator.build_filter(
kindOf='older_than', time_unit='days',
timestring=index_set['time_string'],
value=settings.PRUNE_OLDER_THAN)
# apply the filters to get the final list of indices to delete
working_list = curator.apply_filter(working_list, **name_filter)
working_list = curator.apply_filter(working_list, **age_filter)
if working_list is not None and len(working_list) > 0:
try:
curator.delete_indices(client, working_list)
deleted_indices = deleted_indices + working_list
except Exception:
logger.exception("curator.delete_indices raised an exception")
working_list = all_indices # reset for the next loop iteration
return deleted_indices
def test_es_indices(self, m_conn):
"""Indices tests.
To avoid Elasticsearch calls, we mock out the get_connection call, then
set up additional mocks for the resulting ES connection.
"""
m_es = mock.Mock(Elasticsearch, name='es')
m_indices = mock.MagicMock(IndicesClient, name='indices')
m_es.indices = m_indices
m_es.indices.status.return_value = {
'indices': {
'index1': 'value1',
'not_index1': 'value1',
'index2-': 'value2',
'not_index2-': 'value2'
}
}
m_conn.return_value = m_es
# test with no prefix provided
self.assertEqual(es_indices(conn=es_conn()), "_all")
# test with no params
self.assertEqual(es_indices(), "_all")
# test with no conn, prefix has neither dash nor wildcard
result = es_indices(prefix='index1')
self.assertTrue(m_es.indices.status.called)
self.assertIn('index1', result)
self.assertNotIn('not_index1', result)
# test with prefix which has neither dash nor wildcard
result = es_indices('index1', es_conn())
self.assertIn('index1', result)
self.assertNotIn('not_index1', result)
# test with no conn, prefix has wildcard but no dash
result = es_indices(prefix='index1*')
self.assertTrue(m_es.indices.status.called)
self.assertIn('index1', result)
self.assertNotIn('not_index1', result)
# test with prefix which has wildcard but no dash
result = es_indices('index1*', es_conn())
self.assertIn('index1', result)
self.assertNotIn('not_index1', result)
# test with no conn, prefix has dash but no wildcard
result = es_indices(prefix='index2-')
self.assertTrue(m_es.indices.status.called)
self.assertIn('index2-', result)
self.assertNotIn('not_index2-', result)
# test with prefix which has dash but no wildcard
result = es_indices('index2-', es_conn())
self.assertIn('index2-', result)
self.assertNotIn('not_index2-', result)
# test with no conn, prefix has dash and wildcard
result = es_indices(prefix='index2-*')
self.assertTrue(m_es.indices.status.called)
self.assertIn('index2-', result)
self.assertNotIn('not_index2-', result)
# test with prefix which has dash and wildcard
result = es_indices('index2-*', es_conn())
self.assertIn('index2-', result)
self.assertNotIn('not_index2-', result)
class Meta:
using = es_conn()