Example #1
0
    def test_es_indices(self, m_conn):
        """Indices tests.

        To avoid Elasticsearch calls, we mock out the get_connection call, then
        set up additional mocks for the resulting ES connection.

        """

        m_es = mock.Mock(Elasticsearch, name='es')
        m_indices = mock.MagicMock(IndicesClient, name='indices')
        m_es.indices = m_indices
        m_es.indices.status.return_value = {
            'indices': {
                'index1': 'value1',
                'not_index1': 'value3'
            }
        }
        m_conn.return_value = m_es

        # test with no prefix provided
        self.assertEqual(es_indices(conn=es_conn()), "_all")

        # test with no params
        self.assertEqual(es_indices(), "_all")

        # test with no conn
        result = es_indices(prefix='index')
        self.assertTrue(m_es.indices.status.called)
        self.assertIn('index1', result)
        self.assertNotIn('not_index1', result)

        # test with prefix
        result = es_indices('index', es_conn())
        self.assertIn('index1', result)
        self.assertNotIn('not_index1', result)
Example #2
0
    def test_es_indices(self, m_conn):
        """Indices tests.

        To avoid Elasticsearch calls, we mock out the get_connection call, then
        set up additional mocks for the resulting ES connection.

        """

        m_es = mock.Mock(Elasticsearch, name="es")
        m_indices = mock.MagicMock(IndicesClient, name="indices")
        m_es.indices = m_indices
        m_es.indices.status.return_value = {"indices": {"index1": "value1", "not_index1": "value3"}}
        m_conn.return_value = m_es

        # test with no prefix provided
        self.assertEqual(es_indices(conn=es_conn()), "_all")

        # test with no params
        self.assertEqual(es_indices(), "_all")

        # test with no conn
        result = es_indices(prefix="index")
        self.assertTrue(m_es.indices.status.called)
        self.assertIn("index1", result)
        self.assertNotIn("not_index1", result)

        # test with prefix
        result = es_indices("index", es_conn())
        self.assertIn("index1", result)
        self.assertNotIn("not_index1", result)
Example #3
0
    def test_connection(self, mock_conf, mock_es):
        """Connection tests."""

        mock_es.return_value = None
        mock_conf.return_value = None

        es_conn()
        self.assertEqual(mock_es.call_count, 1)
        mock_conf.assert_called_with(default=settings.ES_SERVER, sniff_on_start=False, max_retries=1)

        mock_conf.reset_mock()
        mock_es.reset_mock()

        es_conn(server={"hosts": ["abc", "def"]})
        self.assertEqual(mock_es.call_count, 1)
        mock_conf.assert_called_with(default={"hosts": ["abc", "def"]}, sniff_on_start=False, max_retries=1)
Example #4
0
def nova_hypervisors_stats():
    """Get stats from the nova API and add them as Goldstone metrics."""
    from goldstone.models import es_conn, daily_index

    novaclient = get_nova_client()
    response = novaclient.hypervisors.statistics()._info
    region = get_keystone_region()
    metric_prefix = 'nova.hypervisor.'
    now = arrow.utcnow()
    conn = es_conn()
    es_index = daily_index(METRIC_INDEX_PREFIX)
    es_doc_type = METRIC_DOCTYPE

    for key, value in response.items():
        doc = {
            'type': es_doc_type,
            'name': metric_prefix + key,
            'value': value,
            'metric_type': 'gauge',
            '@timestamp': now.isoformat(),
            'region': region
        }

        if key in ['disk_available_least', 'free_disk_gb', 'local_gb',
                   'local_gb_used']:
            doc['unit'] = 'GB'
        elif key in ['free_ram_mb', 'memory_mb', 'memory_mb_used']:
            doc['unit'] = 'MB'
        else:
            doc['unit'] = 'count'

        conn.create(es_index, es_doc_type, doc)
Example #5
0
def nova_hypervisors_stats():
    """Get stats from the nova API and add them as Goldstone metrics."""
    from goldstone.utils import get_nova_client
    from goldstone.models import es_conn, daily_index

    novaclient = get_nova_client()['client']
    response = \
        novaclient.hypervisors.statistics()._info     # pylint: disable=W0212
    region = get_region_for_nova_client(novaclient)
    metric_prefix = 'nova.hypervisor.'
    now = arrow.utcnow()
    conn = es_conn()
    es_index = daily_index(MetricData.INDEX_PREFIX)
    es_doc_type = MetricData._doc_type.name      # pylint: disable=W0212

    for key, value in response.items():
        doc = {
            'type': es_doc_type,
            'name': metric_prefix + key,
            'value': value,
            'metric_type': 'gauge',
            '@timestamp': now.isoformat(),
            'region': region
        }

        if key in ['disk_available_least', 'free_disk_gb', 'local_gb',
                   'local_gb_used']:
            doc['unit'] = 'GB'
        elif key in ['free_ram_mb', 'memory_mb', 'memory_mb_used']:
            doc['unit'] = 'MB'
        else:
            doc['unit'] = 'count'

        conn.create(es_index, es_doc_type, doc)
Example #6
0
    def search(cls):
        """Gets a generic Log search object.

        See elasticsearch-dsl for parameter information.
        """

        return Search(
            index=es_indices(cls.INDEX_PREFIX),
            doc_type={cls._doc_type.name: cls.from_es},
        ).sort(cls.SORT).using(es_conn())
Example #7
0
    def test_connection(self, mock_conf, mock_es):
        """Connection tests."""

        mock_es.return_value = None
        mock_conf.return_value = None

        es_conn()
        self.assertEqual(mock_es.call_count, 1)
        mock_conf.assert_called_with(default=settings.ES_SERVER,
                                     sniff_on_start=False,
                                     max_retries=1)

        mock_conf.reset_mock()
        mock_es.reset_mock()

        es_conn(server={'hosts': ['abc', 'def']})
        self.assertEqual(mock_es.call_count, 1)
        mock_conf.assert_called_with(default={'hosts': ['abc', 'def']},
                                     sniff_on_start=False,
                                     max_retries=1)
Example #8
0
    def get_field_mapping(cls, field):
        """Return a field mapping."""

        conn = es_conn()
        index = es_indices(cls.INDEX_PREFIX)

        return conn.indices.get_field_mapping(field,
                                              index,
                                              cls._doc_type.name,
                                              include_defaults=True,
                                              allow_no_indices=False)
Example #9
0
    def save(self, using=None, index=None, **kwargs):
        """Posts a record to the database.

        See elasticsearch-dsl for parameter information.
        """
        if using is None:
            using = es_conn()

        if index is None:
            index = daily_index(self.INDEX_PREFIX)

        return super(DailyIndexDocType, self).save(using, index, **kwargs)
Example #10
0
def prune_es_indices():
    """Prune ES indices older than the age defined in
    settings.PRUNE_OLDER_THAN."""

    curation_params = [
        {"prefix": "events_", "time_string": "%Y-%m-%d"},
        {"prefix": "logstash-", "time_string": "%Y.%m.%d"},
        {"prefix": "goldstone-", "time_string": "%Y.%m.%d"},
        {"prefix": "goldstone_metrics-", "time_string": "%Y.%m.%d"},
        {"prefix": "api_stats-", "time_string": "%Y.%m.%d"},
        {"prefix": "internal-", "time_string": "%Y.%m.%d"},
    ]

    client = es_conn()
    all_indices = curator.get_indices(client)
    deleted_indices = []
    working_list = all_indices  # we'll whittle this down with filters

    for index_set in curation_params:

        # filter on our prefix
        name_filter = curator.build_filter(
            kindOf='prefix', value=index_set['prefix'])

        # filter on the datestring
        age_filter = curator.build_filter(
            kindOf='older_than', time_unit='days',
            timestring=index_set['time_string'],
            value=settings.PRUNE_OLDER_THAN)

        # apply the filters to get the final list of indices to delete
        working_list = curator.apply_filter(working_list, **name_filter)
        working_list = curator.apply_filter(working_list, **age_filter)

        if working_list is not None and len(working_list) > 0:
            try:
                curator.delete_indices(client, working_list)
                deleted_indices = deleted_indices + working_list
            except Exception:
                logger.exception("curator.delete_indices raised an exception")

        working_list = all_indices  # reset for the next loop iteration

    return deleted_indices
Example #11
0
def prune_es_indices():
    """Prune ES indices older than the age defined in
    settings.PRUNE_OLDER_THAN."""

    curation_params = [
        {"prefix": "events_", "time_string": "%Y-%m-%d"},
        {"prefix": "logstash-", "time_string": "%Y.%m.%d"},
        {"prefix": "goldstone-", "time_string": "%Y.%m.%d"},
        {"prefix": "goldstone_metrics-", "time_string": "%Y.%m.%d"},
        {"prefix": "api_stats-", "time_string": "%Y.%m.%d"},
        {"prefix": "internal-", "time_string": "%Y.%m.%d"},
    ]

    client = es_conn()
    all_indices = curator.get_indices(client)
    deleted_indices = []
    working_list = all_indices  # we'll whittle this down with filters

    for index_set in curation_params:

        # filter on our prefix
        name_filter = curator.build_filter(
            kindOf='prefix', value=index_set['prefix'])

        # filter on the datestring
        age_filter = curator.build_filter(
            kindOf='older_than', time_unit='days',
            timestring=index_set['time_string'],
            value=settings.PRUNE_OLDER_THAN)

        # apply the filters to get the final list of indices to delete
        working_list = curator.apply_filter(working_list, **name_filter)
        working_list = curator.apply_filter(working_list, **age_filter)

        if working_list is not None and len(working_list) > 0:
            try:
                curator.delete_indices(client, working_list)
                deleted_indices = deleted_indices + working_list
            except Exception:
                logger.exception("curator.delete_indices raised an exception")

        working_list = all_indices  # reset for the next loop iteration

    return deleted_indices
Example #12
0
    def test_es_indices(self, m_conn):
        """Indices tests.

        To avoid Elasticsearch calls, we mock out the get_connection call, then
        set up additional mocks for the resulting ES connection.

        """

        m_es = mock.Mock(Elasticsearch, name='es')
        m_indices = mock.MagicMock(IndicesClient, name='indices')
        m_es.indices = m_indices
        m_es.indices.status.return_value = {
            'indices': {
                'index1': 'value1',
                'not_index1': 'value1',
                'index2-': 'value2',
                'not_index2-': 'value2'
            }
        }

        m_conn.return_value = m_es

        # test with no prefix provided
        self.assertEqual(es_indices(conn=es_conn()), "_all")

        # test with no params
        self.assertEqual(es_indices(), "_all")

        # test with no conn, prefix has neither dash nor wildcard
        result = es_indices(prefix='index1')
        self.assertTrue(m_es.indices.status.called)
        self.assertIn('index1', result)
        self.assertNotIn('not_index1', result)

        # test with prefix which has neither dash nor wildcard
        result = es_indices('index1', es_conn())
        self.assertIn('index1', result)
        self.assertNotIn('not_index1', result)

        # test with no conn, prefix has wildcard but no dash
        result = es_indices(prefix='index1*')
        self.assertTrue(m_es.indices.status.called)
        self.assertIn('index1', result)
        self.assertNotIn('not_index1', result)

        # test with prefix which has wildcard but no dash
        result = es_indices('index1*', es_conn())
        self.assertIn('index1', result)
        self.assertNotIn('not_index1', result)

        # test with no conn, prefix has dash but no wildcard
        result = es_indices(prefix='index2-')
        self.assertTrue(m_es.indices.status.called)
        self.assertIn('index2-', result)
        self.assertNotIn('not_index2-', result)

        # test with prefix which has dash but no wildcard
        result = es_indices('index2-', es_conn())
        self.assertIn('index2-', result)
        self.assertNotIn('not_index2-', result)

        # test with no conn, prefix has dash and wildcard
        result = es_indices(prefix='index2-*')
        self.assertTrue(m_es.indices.status.called)
        self.assertIn('index2-', result)
        self.assertNotIn('not_index2-', result)

        # test with prefix which has dash and wildcard
        result = es_indices('index2-*', es_conn())
        self.assertIn('index2-', result)
        self.assertNotIn('not_index2-', result)
Example #13
0
 class Meta:
     using = es_conn()