def setUp(self):
     self.maxDiff = None
     self.es = ElasticSearch("http://url.com",
                             AppFilter(app="app_name").query())
     self.index = ".measures-tsuru-{}*".format(
         datetime.datetime.utcnow().strftime("%Y.%m.%d"))
 def setUp(self):
     self.maxDiff = None
     self.es = ElasticSearch("http://url.com", AppFilter(app="app_name").query())
     self.index = ".measure-tsuru-{}*".format(datetime.datetime.utcnow().strftime("%Y.%m.%d"))
class ElasticSearchTest(TestCase):
    def setUp(self):
        self.maxDiff = None
        self.es = ElasticSearch("http://url.com",
                                AppFilter(app="app_name").query())
        self.index = ".measures-tsuru-{}*".format(
            datetime.datetime.utcnow().strftime("%Y.%m.%d"))

    @patch("requests.post")
    def test_cpu_max(self, post_mock):
        self.es.process = Mock()
        self.es.cpu_max()
        url = "{}/{}/_search".format(self.es.url, self.index)
        query = self.es.query("cpu_max")
        query["query"]["bool"]["filter"].append(
            {"range": {
                "value": {
                    "lt": 500
                }
            }})
        post_mock.assert_called_with(
            url,
            headers={'Content-Type': 'application/json'},
            data=json.dumps(query))

    @patch("requests.post")
    def test_mem_max(self, post_mock):
        self.es.process = Mock()
        self.es.mem_max()
        url = "{}/{}/_search".format(self.es.url, self.index)
        post_mock.assert_called_with(
            url,
            headers={'Content-Type': 'application/json'},
            data=json.dumps(self.es.query("mem_max")))

    @patch("requests.post")
    def test_swap(self, post_mock):
        self.es.process = Mock()
        self.es.swap()
        url = "{}/{}/_search".format(self.es.url, self.index)
        post_mock.assert_called_with(
            url,
            headers={'Content-Type': 'application/json'},
            data=json.dumps(self.es.query("swap")))

    @patch("requests.post")
    def test_units(self, post_mock):
        self.es.units()
        url = "{}/{}/_search".format(self.es.url, self.index)
        aggregation = {"units": {"cardinality": {"field": "host.keyword"}}}
        post_mock.assert_called_with(
            url,
            headers={'Content-Type': 'application/json'},
            data=json.dumps(self.es.query("cpu_max", aggregation=aggregation)))

    @patch("requests.post")
    def test_requests_min(self, post_mock):
        self.es.requests_min()
        url = "{}/{}/_search".format(self.es.url, self.index)
        aggregation = {"sum": {"sum": {"field": "count"}}}
        post_mock.assert_called_with(
            url,
            headers={'Content-Type': 'application/json'},
            data=json.dumps(
                self.es.query("response_time", aggregation=aggregation)))

    @patch("requests.post")
    def test_response_time(self, post_mock):
        self.es.response_time()
        url = "{}/{}/_search".format(self.es.url, self.index)
        aggregation = {
            "stats": {
                "stats": {
                    "field": "value"
                }
            },
            "percentiles": {
                "percentiles": {
                    "field": "value"
                }
            }
        }
        post_mock.assert_called_with(
            url,
            headers={'Content-Type': 'application/json'},
            data=json.dumps(
                self.es.query("response_time", aggregation=aggregation)))

    @patch("requests.post")
    def test_http_methods(self, post_mock):
        self.es.http_methods()
        url = "{}/{}/_search".format(self.es.url, self.index)
        aggregation = {"method": {"terms": {"field": "method"}}}
        post_mock.assert_called_with(
            url,
            headers={'Content-Type': 'application/json'},
            data=json.dumps(
                self.es.query("response_time", aggregation=aggregation)))

    @patch("requests.post")
    def test_status_code(self, post_mock):
        self.es.status_code()
        url = "{}/{}/_search".format(self.es.url, self.index)
        aggregation = {"status_code": {"terms": {"field": "status_code"}}}
        post_mock.assert_called_with(
            url,
            headers={'Content-Type': 'application/json'},
            data=json.dumps(
                self.es.query("response_time", aggregation=aggregation)))

    @patch("requests.post")
    def test_top_slow(self, post_mock):
        self.es.top_slow()
        url = "{}/{}/_search".format(self.es.url, self.index)
        query = {
            "query":
            self.es.add_metric_filter(self.es.filtered_query, "response_time"),
            "size":
            0,
            "aggs": {
                "top": {
                    "terms": {
                        "script":
                        "doc['method'].value +'|-o-|'+doc['path.keyword'].value +'|-o-|'+doc['status_code'].value",
                        "order": {
                            "stats.max": "desc"
                        }
                    },
                    "aggs": {
                        "stats": {
                            "stats": {
                                "field": "value"
                            }
                        },
                        "percentiles": {
                            "percentiles": {
                                "field": "value"
                            }
                        },
                        "max": {
                            "top_hits": {
                                "sort": [{
                                    "value": {
                                        "order": "desc"
                                    }
                                }],
                                "size": 1
                            }
                        }
                    }
                }
            }
        }
        post_mock.assert_called_with(
            url,
            headers={'Content-Type': 'application/json'},
            data=json.dumps(query))

    @patch("requests.post")
    def test_connections(self, post_mock):
        self.es.connections()
        url = "{}/{}/_search".format(self.es.url, self.index)
        aggregation = {
            "connection": {
                "terms": {
                    "field": "connection.keyword"
                }
            }
        }
        post_mock.assert_called_with(
            url,
            headers={'Content-Type': 'application/json'},
            data=json.dumps(
                self.es.query("connection", aggregation=aggregation)))

    @patch("requests.post")
    def test_netrx(self, post_mock):
        self.es.netrx()
        url = "{}/{}/_search".format(self.es.url, self.index)
        post_mock.assert_called_with(
            url,
            headers={'Content-Type': 'application/json'},
            data=json.dumps(self.es.query("netrx",
                                          aggregation=NET_AGGREGATION)))

    @patch("requests.post")
    def test_nettx(self, post_mock):
        self.es.nettx()
        url = "{}/{}/_search".format(self.es.url, self.index)
        post_mock.assert_called_with(
            url,
            headers={'Content-Type': 'application/json'},
            data=json.dumps(self.es.query("nettx",
                                          aggregation=NET_AGGREGATION)))

    def test_process(self):
        data = {
            "took": 86,
            "timed_out": False,
            "_shards": {
                "total": 266,
                "successful": 266,
                "failed": 0
            },
            "hits": {
                "total": 644073,
                "max_score": 0,
                "hits": []
            },
            "aggregations": {
                "date": {
                    "buckets": [{
                        "key_as_string": "2015-07-21T19:35:00.000Z",
                        "key": 1437507300000,
                        "doc_count": 9,
                        "stats": {
                            "min": 97517568,
                            "max": 97517568,
                            "avg": 97517568
                        }
                    }, {
                        "key_as_string": "2015-07-21T19:36:00.000Z",
                        "key": 1437507360000,
                        "doc_count": 9,
                        "stats": {
                            "min": 97517568,
                            "max": 97517568,
                            "avg": 97517568
                        }
                    }]
                }
            }
        }
        expected = {
            "data": {
                "max": [[1437507300000, 97517568], [1437507360000, 97517568]],
                "min": [[1437507300000, 97517568], [1437507360000, 97517568]],
                "avg": [[1437507300000, 97517568], [1437507360000, 97517568]],
            },
            "min": 97517568,
            "max": 97517569
        }
        d = self.es.process(data)
        self.assertDictEqual(d, expected)

    def test_process_custom_formatter(self):
        data = {
            "took": 86,
            "timed_out": False,
            "_shards": {
                "total": 266,
                "successful": 266,
                "failed": 0
            },
            "hits": {
                "total": 644073,
                "max_score": 0,
                "hits": []
            },
            "aggregations": {
                "date": {
                    "buckets": [{
                        "key_as_string": "2015-07-21T19:35:00.000Z",
                        "key": 1437507300000,
                        "doc_count": 9,
                        "stats": {
                            "min": 97517568,
                            "max": 97517568,
                            "avg": 97517568
                        }
                    }, {
                        "key_as_string": "2015-07-21T19:36:00.000Z",
                        "key": 1437507360000,
                        "doc_count": 9,
                        "stats": {
                            "min": 97517568,
                            "max": 97517568,
                            "avg": 97517568
                        }
                    }]
                }
            }
        }
        expected = {
            "data": {
                "max": [[1437507300000, 93], [1437507360000, 93]],
                "min": [[1437507300000, 93], [1437507360000, 93]],
                "avg": [[1437507300000, 93], [1437507360000, 93]],
            },
            "min": 93,
            "max": 94
        }
        d = self.es.process(data, formatter=lambda x: x / (1024 * 1024))
        self.assertDictEqual(d, expected)

    def test_process_no_aggregation(self):
        data = {
            "took": 1,
            "timed_out": False,
            "_shards": {
                "total": 0,
                "successful": 0,
                "failed": 0
            },
            "hits": {
                "total": 0,
                "max_score": 0,
                "hits": []
            }
        }
        expected = {'data': {}, 'max': 1, 'min': 0}
        d = self.es.process(data, formatter=lambda x: x / (1024 * 1024))
        self.assertDictEqual(d, expected)

    def test_unit_process_no_aggregation(self):
        data = {
            "took": 1,
            "timed_out": False,
            "_shards": {
                "total": 0,
                "successful": 0,
                "failed": 0
            },
            "hits": {
                "total": 0,
                "max_score": 0,
                "hits": []
            }
        }
        expected = {'data': {}, 'max': 1, 'min': 0}
        d = self.es.base_process(data, self.es.units_process)
        self.assertDictEqual(d, expected)

    def test_request_min_process_no_aggregation(self):
        data = {
            "took": 1,
            "timed_out": False,
            "_shards": {
                "total": 0,
                "successful": 0,
                "failed": 0
            },
            "hits": {
                "total": 0,
                "max_score": 0,
                "hits": []
            }
        }
        expected = {'data': {}, 'max': 1, 'min': 0}
        d = self.es.base_process(data, self.es.requests_min_process)
        self.assertDictEqual(d, expected)

    def test_connections_process_no_aggregation(self):
        data = {
            "took": 1,
            "timed_out": False,
            "_shards": {
                "total": 0,
                "successful": 0,
                "failed": 0
            },
            "hits": {
                "total": 0,
                "max_score": 0,
                "hits": []
            }
        }
        expected = {'data': {}, 'max': 1, 'min': 0}
        d = self.es.base_process(data, self.es.connections_process)
        self.assertDictEqual(d, expected)
class ElasticSearchTest(TestCase):
    def setUp(self):
        self.maxDiff = None
        self.es = ElasticSearch("http://url.com", AppFilter(app="app_name").query())
        self.index = ".measure-tsuru-{}*".format(datetime.datetime.utcnow().strftime("%Y.%m.%d"))

    @patch("requests.post")
    def test_cpu_max(self, post_mock):
        self.es.process = Mock()
        self.es.cpu_max()
        url = "{}/{}/{}/_search".format(self.es.url, self.index, "cpu_max")
        post_mock.assert_called_with(url, data=json.dumps(self.es.query()))

    @patch("requests.post")
    def test_mem_max(self, post_mock):
        self.es.process = Mock()
        self.es.mem_max()
        url = "{}/{}/{}/_search".format(self.es.url, self.index, "mem_max")
        post_mock.assert_called_with(url, data=json.dumps(self.es.query()))

    @patch("requests.post")
    def test_swap(self, post_mock):
        self.es.process = Mock()
        self.es.swap()
        url = "{}/{}/{}/_search".format(self.es.url, self.index, "swap")
        post_mock.assert_called_with(url, data=json.dumps(self.es.query()))

    @patch("requests.post")
    def test_units(self, post_mock):
        self.es.units()
        url = "{}/{}/{}/_search".format(self.es.url, self.index, "cpu_max")
        aggregation = {"units": {"cardinality": {"field": "host.raw"}}}
        post_mock.assert_called_with(url, data=json.dumps(self.es.query(aggregation=aggregation)))

    @patch("requests.post")
    def test_requests_min(self, post_mock):
        self.es.requests_min()
        url = "{}/{}/{}/_search".format(self.es.url, self.index, "response_time")
        aggregation = {"sum": {"sum": {"field": "count"}}}
        post_mock.assert_called_with(url, data=json.dumps(self.es.query(aggregation=aggregation)))

    @patch("requests.post")
    def test_response_time(self, post_mock):
        self.es.response_time()
        url = "{}/{}/{}/_search".format(self.es.url, self.index, "response_time")
        aggregation = {
            "stats": {"stats": {"field": "value"}},
            "percentiles": {"percentiles": {"field": "value"}}
        }
        post_mock.assert_called_with(url, data=json.dumps(self.es.query(aggregation=aggregation)))

    @patch("requests.post")
    def test_http_methods(self, post_mock):
        self.es.http_methods()
        url = "{}/{}/{}/_search".format(self.es.url, self.index, "response_time")
        aggregation = {"method": {"terms": {"field": "method"}}}
        post_mock.assert_called_with(url, data=json.dumps(self.es.query(aggregation=aggregation)))

    @patch("requests.post")
    def test_status_code(self, post_mock):
        self.es.status_code()
        url = "{}/{}/{}/_search".format(self.es.url, self.index, "response_time")
        aggregation = {"status_code": {"terms": {"field": "status_code"}}}
        post_mock.assert_called_with(url, data=json.dumps(self.es.query(aggregation=aggregation)))

    @patch("requests.post")
    def test_top_slow(self, post_mock):
        self.es.top_slow()
        url = "{}/{}/{}/_search".format(self.es.url, self.index, "response_time")
        query = {
            "query": self.es.filtered_query,
            "size": 0,
            "aggs": {
                "top": {
                    "terms": {
                        "script": "doc['method'].value +'|-o-|'+doc['path.raw'].value +'|-o-|'+doc['status_code'].value",
                        "order": {
                            "stats.max": "desc"
                        }
                    },
                    "aggs": {
                        "stats": {"stats": {"field": "value"}},
                        "percentiles": {"percentiles": {"field": "value"}},
                        "max": {
                            "top_hits": {
                                "sort": [{"value": {"order": "desc"}}],
                                "size": 1
                            }
                        }
                    }
                }
            }
        }
        post_mock.assert_called_with(url, data=json.dumps(query))

    @patch("requests.post")
    def test_connections(self, post_mock):
        self.es.connections()
        url = "{}/{}/{}/_search".format(self.es.url, self.index, "connection")
        aggregation = {"connection": {"terms": {"field": "connection.raw"}}}
        post_mock.assert_called_with(url, data=json.dumps(self.es.query(aggregation=aggregation)))

    @patch("requests.post")
    def test_netrx(self, post_mock):
        self.es.netrx()
        url = "{}/{}/{}/_search".format(self.es.url, self.index, "netrx")
        post_mock.assert_called_with(url, data=json.dumps(self.es.query(aggregation=NET_AGGREGATION)))

    @patch("requests.post")
    def test_nettx(self, post_mock):
        self.es.nettx()
        url = "{}/{}/{}/_search".format(self.es.url, self.index, "nettx")
        post_mock.assert_called_with(url, data=json.dumps(self.es.query(aggregation=NET_AGGREGATION)))

    def test_process(self):
        data = {
            "took": 86,
            "timed_out": False,
            "_shards": {
                "total": 266,
                "successful": 266,
                "failed": 0
            },
            "hits": {
                "total": 644073,
                "max_score": 0,
                "hits": []
            },
            "aggregations": {
                "date": {
                    "buckets": [
                        {
                            "key_as_string": "2015-07-21T19:35:00.000Z",
                            "key": 1437507300000,
                            "doc_count": 9,
                            "stats": {
                                "min": 97517568,
                                "max": 97517568,
                                "avg": 97517568
                            }
                        },
                        {
                            "key_as_string": "2015-07-21T19:36:00.000Z",
                            "key": 1437507360000,
                            "doc_count": 9,
                            "stats": {
                                "min": 97517568,
                                "max": 97517568,
                                "avg": 97517568
                            }
                        }
                    ]
                }
            }
        }
        expected = {
            "data": {
                "max": [[1437507300000, 97517568], [1437507360000, 97517568]],
                "min": [[1437507300000, 97517568], [1437507360000, 97517568]],
                "avg": [[1437507300000, 97517568], [1437507360000, 97517568]],
            },
            "min": 97517568,
            "max": 97517569
        }
        d = self.es.process(data)
        self.assertDictEqual(d, expected)

    def test_process_custom_formatter(self):
        data = {
            "took": 86,
            "timed_out": False,
            "_shards": {
                "total": 266,
                "successful": 266,
                "failed": 0
            },
            "hits": {
                "total": 644073,
                "max_score": 0,
                "hits": []
            },
            "aggregations": {
                "date": {
                    "buckets": [
                        {
                            "key_as_string": "2015-07-21T19:35:00.000Z",
                            "key": 1437507300000,
                            "doc_count": 9,
                            "stats": {
                                "min": 97517568,
                                "max": 97517568,
                                "avg": 97517568
                            }
                        },
                        {
                            "key_as_string": "2015-07-21T19:36:00.000Z",
                            "key": 1437507360000,
                            "doc_count": 9,
                            "stats": {
                                "min": 97517568,
                                "max": 97517568,
                                "avg": 97517568
                            }
                        }
                    ]
                }
            }
        }
        expected = {
            "data": {
                "max": [[1437507300000, 93], [1437507360000, 93]],
                "min": [[1437507300000, 93], [1437507360000, 93]],
                "avg": [[1437507300000, 93], [1437507360000, 93]],
            },
            "min": 93,
            "max": 94
        }
        d = self.es.process(data, formatter=lambda x: x / (1024 * 1024))
        self.assertDictEqual(d, expected)

    def test_process_no_aggregation(self):
        data = {
            "took": 1,
            "timed_out": False,
            "_shards": {
                "total": 0,
                "successful": 0,
                "failed": 0
            },
            "hits": {
                "total": 0,
                "max_score": 0,
                "hits": []
            }
        }
        expected = {'data': {}, 'max': 1, 'min': 0}
        d = self.es.process(data, formatter=lambda x: x / (1024 * 1024))
        self.assertDictEqual(d, expected)

    def test_unit_process_no_aggregation(self):
        data = {
            "took": 1,
            "timed_out": False,
            "_shards": {
                "total": 0,
                "successful": 0,
                "failed": 0
            },
            "hits": {
                "total": 0,
                "max_score": 0,
                "hits": []
            }
        }
        expected = {'data': {}, 'max': 1, 'min': 0}
        d = self.es.base_process(data, self.es.units_process)
        self.assertDictEqual(d, expected)

    def test_request_min_process_no_aggregation(self):
        data = {
            "took": 1,
            "timed_out": False,
            "_shards": {
                "total": 0,
                "successful": 0,
                "failed": 0
            },
            "hits": {
                "total": 0,
                "max_score": 0,
                "hits": []
            }
        }
        expected = {'data': {}, 'max': 1, 'min': 0}
        d = self.es.base_process(data, self.es.requests_min_process)
        self.assertDictEqual(d, expected)

    def test_connections_process_no_aggregation(self):
        data = {
            "took": 1,
            "timed_out": False,
            "_shards": {
                "total": 0,
                "successful": 0,
                "failed": 0
            },
            "hits": {
                "total": 0,
                "max_score": 0,
                "hits": []
            }
        }
        expected = {'data': {}, 'max': 1, 'min': 0}
        d = self.es.base_process(data, self.es.connections_process)
        self.assertDictEqual(d, expected)