Ejemplo n.º 1
0
    def test_variables_with_numbers(self):
        """
        Account for nginx variables with numbers in them
        """
        log_format = '$remote_addr [$time_local] $status $geoip_country_code ' + \
                     '$geoip_country_code3 "$geoip_country_name"'

        lines = [
            '10.10.10.102 [29/Jun/2016:23:31:07 +0000] 200 US USA "United States"',
            '10.10.10.46 [29/Jun/2016:23:34:33 +0000] 200 CA CAN "Canada"',
            '10.10.10.189 [29/Jun/2016:23:34:42 +0000] 200 IE IRL "Ireland"',
            '10.10.10.194 [29/Jun/2016:23:35:02 +0000] 200 NL NLD "Netherlands"',
            '10.10.10.198 [29/Jun/2016:23:37:08 +0000] 200 FR FRA "France"',
            '10.10.10.232 [29/Jun/2016:23:37:49 +0000] 200 SG SGP "Singapore"',
            '10.10.10.100 [29/Jun/2016:23:38:19 +0000] 200 ID IDN "Indonesia"'
        ]

        collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines)
        collector.collect()

        # Make sure that variable name with number is properly formatted...
        keys = collector.parser.keys
        assert_that(keys, has_item('geoip_country_code'))
        assert_that(keys, has_item('geoip_country_code3'))

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))

        # check some values
        counter = metrics['counter']
        assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(7))
Ejemplo n.º 2
0
    def test_server_name(self):
        self.fake_object.filters = [
            Filter(filter_rule_id=2,
                   metric='nginx.http.status.2xx',
                   data=[['$server_name', '~', 'differentsimgirls.com']])
        ]

        collector = NginxAccessLogsCollector(
            object=self.fake_object,
            log_format=
            '$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent '
            +
            '\"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\" '
            +
            'rt=$request_time ua=\"$upstream_addr\" us=\"$upstream_status\" ' +
            'ut=\"$upstream_response_time\" ul=\"$upstream_response_length\" '
            + 'cs=$upstream_cache_status sn=$server_name',
            tail=[
                '104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 200 28275 "-" '
                +
                '"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="-" ul="-" cs=- sn=differentsimgirls.com'
            ])
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))
        counter = metrics['counter']

        # check our metric
        assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(1))
        assert_that(counter['C|nginx.http.status.2xx||2'][0][1], equal_to(1))
Ejemplo n.º 3
0
    def test_regex_filter(self):
        self.fake_object.filters = [
            Filter(filter_rule_id=2,
                   metric='nginx.http.request.body_bytes_sent',
                   data=[['$request_uri', '~', '/img.*'],
                         ['$server_protocol', '~', 'HTTP/1.2']])
        ]

        collector = NginxAccessLogsCollector(object=self.fake_object,
                                             tail=self.lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))

        # counters
        counter = metrics['counter']
        for key in ('C|nginx.http.method.get',
                    'C|nginx.http.request.body_bytes_sent',
                    'C|nginx.http.status.3xx', 'C|nginx.http.status.2xx',
                    'C|nginx.http.method.post', 'C|nginx.http.v1_1',
                    'C|nginx.http.status.4xx',
                    'C|nginx.http.request.body_bytes_sent||2',
                    'C|nginx.http.method.post'):
            assert_that(counter, has_key(key))

        # values
        assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(5))
        assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(2))
        assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(3))

        # filter values
        assert_that(counter['C|nginx.http.request.body_bytes_sent||2'][0][1],
                    equal_to(2))
Ejemplo n.º 4
0
    def test_unused_filter_defaults_zero(self):
        self.fake_object.filters = [
            Filter(filter_rule_id=1, metric='nginx.http.status.4xx', data=[{'status': '416'}]),
            Filter(filter_rule_id=2, metric='nginx.http.status.4xx', data=[{'status': '300'}]),
            Filter(filter_rule_id=3, metric='nginx.http.method.get', data=[{'$request_uri': '/img.*'}]),
            Filter(filter_rule_id=4, metric='nginx.http.method.get', data=[{'$request_uri': 'abcdef'}]),
        ]

        collector = NginxAccessLogsCollector(object=self.fake_object, tail=self.lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))

        # counters
        counter = metrics['counter']

        # filter values
        assert_that(counter, has_key('C|nginx.http.status.4xx||1'))
        assert_that(counter, has_key('C|nginx.http.status.4xx||2'))
        assert_that(counter, has_key('C|nginx.http.method.get||3'))
        assert_that(counter, has_key('C|nginx.http.method.get||4'))
        assert_that(counter['C|nginx.http.status.4xx||1'], contains(contains(is_(int), 1)))
        assert_that(counter['C|nginx.http.status.4xx||2'], contains(contains(is_(int), 0)))
        assert_that(counter['C|nginx.http.method.get||3'], contains(contains(is_(int), 2)))
        assert_that(counter['C|nginx.http.method.get||4'], contains(contains(is_(int), 0)))
Ejemplo n.º 5
0
    def test_variables_with_numbers(self):
        """
        Account for nginx variables with numbers in them
        """
        log_format = '$remote_addr [$time_local] $status $geoip_country_code ' + \
                     '$geoip_country_code3 "$geoip_country_name"'

        lines = [
            '10.10.10.102 [29/Jun/2016:23:31:07 +0000] 200 US USA "United States"',
            '10.10.10.46 [29/Jun/2016:23:34:33 +0000] 200 CA CAN "Canada"',
            '10.10.10.189 [29/Jun/2016:23:34:42 +0000] 200 IE IRL "Ireland"',
            '10.10.10.194 [29/Jun/2016:23:35:02 +0000] 200 NL NLD "Netherlands"',
            '10.10.10.198 [29/Jun/2016:23:37:08 +0000] 200 FR FRA "France"',
            '10.10.10.232 [29/Jun/2016:23:37:49 +0000] 200 SG SGP "Singapore"',
            '10.10.10.100 [29/Jun/2016:23:38:19 +0000] 200 ID IDN "Indonesia"'
        ]

        collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines)
        collector.collect()

        # Make sure that variable name with number is properly formatted...
        regex_string = collector.parser.regex_string
        assert_that(regex_string, contains_string('<geoip_country_code>'))
        assert_that(regex_string, contains_string('<geoip_country_code3>'))

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))

        # check some values
        counter = metrics['counter']
        assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(7))
Ejemplo n.º 6
0
    def test_upstreams(self):
        log_format = '$remote_addr - $remote_user [$time_local] ' + \
                     '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
                     'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status'

        line = \
            '1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \
            '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS'

        # run single method
        collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
        collector.upstreams(NginxAccessLogParser(log_format).parse(line))

        # check
        metrics = self.fake_object.statsd.current
        assert_that(metrics, has_item('counter'))
        assert_that(metrics, has_item('timer'))

        # counters
        counters = metrics['counter']
        assert_that(counters, has_item('nginx.upstream.request.count'))
        assert_that(counters, has_item('nginx.upstream.next.count'))
        assert_that(counters, has_item('nginx.cache.miss'))
        assert_that(counters['nginx.upstream.request.count'][0][1],
                    equal_to(1))
        assert_that(counters['nginx.upstream.next.count'][0][1], equal_to(1))
        assert_that(counters['nginx.cache.miss'][0][1], equal_to(1))

        # histogram
        histogram = metrics['timer']
        assert_that(histogram, has_item('nginx.upstream.response.time'))
        assert_that(histogram['nginx.upstream.response.time'],
                    equal_to([2.001 + 0.345]))
Ejemplo n.º 7
0
    def test_combined(self):
        lines = [
            '178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /img/docker.png HTTP/1.1" 304 0 '
            +
            '"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' +
            '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) '
            + 'Chrome/43.0.2357.124 Safari/537.36"',
            '178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /api/inventory/objects/ HTTP/1.1" 200 1093 '
            +
            '"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' +
            '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) '
            + 'Chrome/43.0.2357.124 Safari/537.36"',
            '127.0.0.1 - - [18/Jun/2015:17:22:33 +0000] "POST /1.0/589fjinijenfirjf/meta/ HTTP/1.1" '
            +
            '202 2 "-" "python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"',
            '52.6.158.18 - - [18/Jun/2015:17:22:40 +0000] "GET /#/objects HTTP/1.1" 416 84 '
            +
            '"-" "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)"'
        ]

        collector = NginxAccessLogsCollector(object=self.fake_object,
                                             tail=lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))

        # counters
        counter = metrics['counter']
        for key in ('C|nginx.http.method.get',
                    'C|nginx.http.request.body_bytes_sent',
                    'C|nginx.http.status.3xx', 'C|nginx.http.status.2xx',
                    'C|nginx.http.method.post', 'C|nginx.http.v1_1',
                    'C|nginx.http.status.4xx'):
            assert_that(counter, has_key(key))

        # values
        assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(3))
        assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2))
        assert_that(counter['C|nginx.http.v1_1'][0][1], equal_to(4))
        assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1],
                    equal_to(84 + 2 + 1093 + 0))

        # check zero values
        for counter_name, counter_key in collector.counters.iteritems():
            if counter_key in collector.parser.keys:
                assert_that(counter, has_key('C|%s' % counter_name))
                if counter_name not in ('nginx.http.method.get',
                                        'nginx.http.method.post',
                                        'nginx.http.status.2xx',
                                        'nginx.http.status.3xx',
                                        'nginx.http.status.4xx',
                                        'nginx.http.v1_1',
                                        'nginx.http.request.body_bytes_sent'):
                    assert_that(counter['C|%s' % counter_name][0][1],
                                equal_to(0))
            elif counter_key is not None:
                if counter_key not in collector.parser.request_variables:
                    assert_that(counter, not_(has_key('C|%s' % counter_name)))
Ejemplo n.º 8
0
    def test_server_name(self):
        self.fake_object.filters = [
            Filter(**dict(
                filter_rule_id=2,
                metric='nginx.http.status.2xx',
                data=[
                    {u'$server_name': u'differentsimgirls.com'}
                ]
            ))
        ]

        collector = NginxAccessLogsCollector(
            object=self.fake_object,
            log_format='$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent ' +
                       '\"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\" ' +
                       'rt=$request_time ua=\"$upstream_addr\" us=\"$upstream_status\" ' +
                       'ut=\"$upstream_response_time\" ul=\"$upstream_response_length\" ' +
                       'cs=$upstream_cache_status sn=$server_name',
            tail=[
                '104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 200 28275 "-" ' +
                '"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="-" ul="-" cs=- sn=differentsimgirls.com'
            ]
        )
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))
        counter = metrics['counter']

        # check our metric
        assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(1))
        assert_that(counter['C|nginx.http.status.2xx||2'][0][1], equal_to(1))
Ejemplo n.º 9
0
    def test_separate_4xx_5xx_with_filters(self):
        self.fake_object.filters = [
            Filter(filter_rule_id=1,
                   metric='nginx.http.status.502',
                   data=[
                       ['$request_method', '~', 'GET'],
                   ])
        ]

        collector = NginxAccessLogsCollector(object=self.fake_object,
                                             tail=self.lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))

        # counters
        counter = metrics['counter']
        for key in ('C|nginx.http.method.get',
                    'C|nginx.http.request.body_bytes_sent',
                    'C|nginx.http.status.3xx', 'C|nginx.http.status.2xx',
                    'C|nginx.http.method.post', 'C|nginx.http.v1_1',
                    'C|nginx.http.status.4xx', 'C|nginx.http.status.5xx',
                    'C|nginx.http.status.502||1', 'C|nginx.http.method.post'):
            assert_that(counter, has_key(key))

        # values
        assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(5))
        assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(2))
        assert_that(counter['C|nginx.http.status.5xx'][0][1], equal_to(1))

        # filter values
        assert_that(counter['C|nginx.http.status.502||1'][0][1], equal_to(1))
Ejemplo n.º 10
0
    def test_filter_with_negated(self):
        self.fake_object.filters = [
            Filter(filter_rule_id=1, metric='nginx.http.method.get', data=[['$status', '~', '200']]),
            Filter(filter_rule_id=2, metric='nginx.http.method.get', data=[['$status', '!~', '200']]),
            Filter(filter_rule_id=3, metric='nginx.http.method.get', data=[['$status', '~', '304']]),
            Filter(filter_rule_id=4, metric='nginx.http.method.get', data=[['$status', '!~', '304']]),
        ]

        collector = NginxAccessLogsCollector(object=self.fake_object, tail=self.lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))

        # counters
        counter = metrics['counter']
        assert_that(counter, has_key('C|nginx.http.method.get'))
        assert_that(counter['C|nginx.http.method.get'], contains(contains(is_(int), 4)))

        # filter values
        assert_that(counter, has_key('C|nginx.http.method.get||1'))
        assert_that(counter, has_key('C|nginx.http.method.get||2'))
        assert_that(counter, has_key('C|nginx.http.method.get||3'))
        assert_that(counter, has_key('C|nginx.http.method.get||4'))
        assert_that(counter['C|nginx.http.method.get||1'], contains(contains(is_(int), 2)))
        assert_that(counter['C|nginx.http.method.get||2'], contains(contains(is_(int), 2)))
        assert_that(counter['C|nginx.http.method.get||3'], contains(contains(is_(int), 1)))
        assert_that(counter['C|nginx.http.method.get||4'], contains(contains(is_(int), 3)))
Ejemplo n.º 11
0
    def test_unused_filter_defaults_zero(self):
        self.fake_object.filters = [
            Filter(filter_rule_id=1, metric='nginx.http.status.4xx', data=[['status', '~', '416']]),
            Filter(filter_rule_id=2, metric='nginx.http.status.4xx', data=[['status', '~', '300']]),
            Filter(filter_rule_id=3, metric='nginx.http.method.get', data=[['$request_uri', '~', '/img.*']]),
            Filter(filter_rule_id=4, metric='nginx.http.method.get', data=[['$request_uri', '~', 'abcdef']]),
        ]

        collector = NginxAccessLogsCollector(object=self.fake_object, tail=self.lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))

        # counters
        counter = metrics['counter']

        # filter values
        assert_that(counter, has_key('C|nginx.http.status.4xx||1'))
        assert_that(counter, has_key('C|nginx.http.status.4xx||2'))
        assert_that(counter, has_key('C|nginx.http.method.get||3'))
        assert_that(counter, has_key('C|nginx.http.method.get||4'))
        assert_that(counter['C|nginx.http.status.4xx||1'], contains(contains(is_(int), 1)))
        assert_that(counter['C|nginx.http.status.4xx||2'], contains(contains(is_(int), 0)))
        assert_that(counter['C|nginx.http.method.get||3'], contains(contains(is_(int), 2)))
        assert_that(counter['C|nginx.http.method.get||4'], contains(contains(is_(int), 0)))
Ejemplo n.º 12
0
    def test_regex_filter(self):
        self.fake_object.filters = [
            Filter(**dict(
                filter_rule_id=2,
                metric='nginx.http.request.body_bytes_sent',
                data=[
                    {'$request_uri': '/img.*'},
                    {'$server_protocol': 'HTTP/1.2'}
                ]
            ))
        ]

        collector = NginxAccessLogsCollector(object=self.fake_object, tail=self.lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))

        # counters
        counter = metrics['counter']
        for key in ('C|nginx.http.method.get', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.status.3xx',
                    'C|nginx.http.status.2xx', 'C|nginx.http.method.post', 'C|nginx.http.v1_1',
                    'C|nginx.http.status.4xx', 'C|nginx.http.request.body_bytes_sent||2', 'C|nginx.http.method.post'):
            assert_that(counter, has_key(key))

        # values
        assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(4))
        assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(2))
        assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(3))

        # filter values
        assert_that(counter['C|nginx.http.request.body_bytes_sent||2'][0][1], equal_to(2))
Ejemplo n.º 13
0
    def test_upstreams(self):
        log_format = '$remote_addr - $remote_user [$time_local] ' + \
                     '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
                     'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status'

        line = \
            '1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \
            '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS'

        # run single method
        collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
        collector.upstreams(NginxAccessLogParser(log_format).parse(line))

        # check
        metrics = self.fake_object.statsd.current
        assert_that(metrics, has_item('counter'))
        assert_that(metrics, has_item('timer'))

        # counters
        counters = metrics['counter']
        assert_that(counters, has_item('nginx.upstream.request.count'))
        assert_that(counters, has_item('nginx.upstream.next.count'))
        assert_that(counters, has_item('nginx.cache.miss'))
        assert_that(counters['nginx.upstream.request.count'][0][1], equal_to(1))
        assert_that(counters['nginx.upstream.next.count'][0][1], equal_to(1))
        assert_that(counters['nginx.cache.miss'][0][1], equal_to(1))

        # histogram
        histogram = metrics['timer']
        assert_that(histogram, has_item('nginx.upstream.response.time'))
        assert_that(histogram['nginx.upstream.response.time'], equal_to([2.001 + 0.345]))
Ejemplo n.º 14
0
    def test_cache_states(self):
        log_format = '$remote_addr - $remote_user [$time_local] ' + \
                     '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
                     'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status'

        lines = [
            '1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 '
            +
            '"http://www.yandex.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS',
            '1.2.3.4 - - [22/Jan/2010:20:34:21 +0300] "GET /foo/ HTTP/1.1" 300 1078 '
            +
            '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.002" cs=HIT',
            '1.2.3.4 - - [22/Jan/2010:20:34:21 +0300] "GET /foo/ HTTP/1.1" 400 1078 '
            +
            '"http://www.mail.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.002" cs=BLAHBLAHFOO',
        ]

        collector = NginxAccessLogsCollector(object=self.fake_object,
                                             log_format=log_format,
                                             tail=lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))
        assert_that(metrics, has_item('timer'))

        # counter keys
        counter = metrics['counter']
        for key in [
                'C|nginx.http.method.get', 'C|nginx.http.v1_1',
                'C|nginx.upstream.next.count',
                'C|nginx.upstream.request.count', 'C|nginx.http.status.3xx',
                'C|nginx.cache.miss', 'C|nginx.http.status.2xx',
                'C|nginx.http.request.body_bytes_sent', 'C|nginx.cache.hit'
        ]:
            assert_that(counter, has_key(key))

        # timer keys
        timer = metrics['timer']
        for key in [
                'G|nginx.upstream.response.time.pctl95',
                'C|nginx.upstream.response.time.count',
                'C|nginx.http.request.time.count', 'G|nginx.http.request.time',
                'G|nginx.http.request.time.pctl95',
                'G|nginx.http.request.time.median',
                'G|nginx.http.request.time.max',
                'G|nginx.upstream.response.time',
                'G|nginx.upstream.response.time.median',
                'G|nginx.upstream.response.time.max'
        ]:
            assert_that(timer, has_key(key))

        # values
        assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(3))
        assert_that(counter['C|nginx.upstream.request.count'][0][1],
                    equal_to(3))
        assert_that(counter['C|nginx.cache.miss'][0][1], equal_to(1))
        assert_that(counter['C|nginx.cache.hit'][0][1], equal_to(1))
Ejemplo n.º 15
0
    def test_combined(self):
        lines = [
            '178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /img/docker.png HTTP/1.1" 304 0 ' +
            '"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' +
            '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) ' +
            'Chrome/43.0.2357.124 Safari/537.36"',

            '178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /api/inventory/objects/ HTTP/1.1" 200 1093 ' +
            '"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' +
            '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) ' +
            'Chrome/43.0.2357.124 Safari/537.36"',

            '127.0.0.1 - - [18/Jun/2015:17:22:33 +0000] "POST /1.0/589fjinijenfirjf/meta/ HTTP/1.1" ' +
            '202 2 "-" "python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"',

            '52.6.158.18 - - [18/Jun/2015:17:22:40 +0000] "GET /#/objects HTTP/1.1" 416 84 ' +
            '"-" "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)"'
        ]

        collector = NginxAccessLogsCollector(object=self.fake_object, tail=lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))

        # counters
        counter = metrics['counter']
        for key in ('C|nginx.http.method.get', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.status.3xx',
                    'C|nginx.http.status.2xx','C|nginx.http.method.post', 'C|nginx.http.v1_1',
                    'C|nginx.http.status.4xx'):
            assert_that(counter, has_key(key))

        # values
        assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(3))
        assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2))
        assert_that(counter['C|nginx.http.v1_1'][0][1], equal_to(4))
        assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1], equal_to(84 + 2 + 1093 + 0))

        # check zero values
        for counter_name, counter_key in collector.counters.iteritems():
            if counter_key in collector.parser.keys:
                assert_that(counter, has_key('C|%s' % counter_name))
                if counter_name not in (
                    'nginx.http.method.get',
                    'nginx.http.method.post',
                    'nginx.http.status.2xx',
                    'nginx.http.status.3xx',
                    'nginx.http.status.4xx',
                    'nginx.http.v1_1',
                    'nginx.http.request.body_bytes_sent'
                ):
                    assert_that(counter['C|%s' % counter_name][0][1], equal_to(0))
            elif counter_key is not None:
                if counter_key not in collector.parser.request_variables:
                    assert_that(counter, not_(has_key('C|%s' % counter_name)))
Ejemplo n.º 16
0
    def test_http_status(self):
        line = '127.0.0.1 - - [02/Jul/2015:14:49:48 +0000] "GET /basic_status HTTP/1.1" 200 110 "-" ' + \
               '"python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"'

        # run single method
        collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
        collector.http_status(NginxAccessLogParser().parse(line))

        # check
        metrics = self.fake_object.statsd.current
        assert_that(metrics, has_item('counter'))
        counters = metrics['counter']
        assert_that(counters, has_item('nginx.http.status.2xx'))
        assert_that(counters['nginx.http.status.2xx'][0][1], equal_to(1))
Ejemplo n.º 17
0
    def test_http_status(self):
        line = '127.0.0.1 - - [02/Jul/2015:14:49:48 +0000] "GET /basic_status HTTP/1.1" 200 110 "-" ' + \
               '"python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"'

        # run single method
        collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
        collector.http_status(NginxAccessLogParser().parse(line))

        # check
        metrics = self.fake_object.statsd.current
        assert_that(metrics, has_item('counter'))
        counters = metrics['counter']
        assert_that(counters, has_item('nginx.http.status.2xx'))
        assert_that(counters['nginx.http.status.2xx'][0][1], equal_to(1))
Ejemplo n.º 18
0
    def test_cache_states(self):
        log_format = '$remote_addr - $remote_user [$time_local] ' + \
                     '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
                     'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status'

        lines = [
            '1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' +
            '"http://www.yandex.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS',

            '1.2.3.4 - - [22/Jan/2010:20:34:21 +0300] "GET /foo/ HTTP/1.1" 300 1078 ' +
            '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.002" cs=HIT',

            '1.2.3.4 - - [22/Jan/2010:20:34:21 +0300] "GET /foo/ HTTP/1.1" 400 1078 ' +
            '"http://www.mail.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.002" cs=BLAHBLAHFOO',

        ]

        collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))
        assert_that(metrics, has_item('timer'))

        # counter keys
        counter = metrics['counter']
        for key in ['C|nginx.http.method.get', 'C|nginx.http.v1_1', 'C|nginx.upstream.next.count',
                    'C|nginx.upstream.request.count', 'C|nginx.http.status.3xx', 'C|nginx.cache.miss',
                    'C|nginx.http.status.2xx', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.cache.hit']:
            assert_that(counter, has_key(key))

        # timer keys
        timer = metrics['timer']
        for key in ['G|nginx.upstream.response.time.pctl95', 'C|nginx.upstream.response.time.count',
                    'C|nginx.http.request.time.count', 'G|nginx.http.request.time',
                    'G|nginx.http.request.time.pctl95', 'G|nginx.http.request.time.median',
                    'G|nginx.http.request.time.max', 'G|nginx.upstream.response.time',
                    'G|nginx.upstream.response.time.median', 'G|nginx.upstream.response.time.max']:
            assert_that(timer, has_key(key))

        # values
        assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(3))
        assert_that(counter['C|nginx.upstream.request.count'][0][1], equal_to(3))
        assert_that(counter['C|nginx.cache.miss'][0][1], equal_to(1))
        assert_that(counter['C|nginx.cache.hit'][0][1], equal_to(1))
Ejemplo n.º 19
0
    def test_http_status_discarded(self):
        line_template = (
            '127.0.0.1 - - [02/Jul/2015:14:49:48 +0000] "GET /basic_status HTTP/1.1" %d 110 "-" '
            '"python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"')

        # collect requests with $status 400 to 498
        lines = [line_template % x for x in xrange(400, 499)]
        NginxAccessLogsCollector(object=self.fake_object, tail=lines).collect()
        counter = self.fake_object.statsd.flush()['metrics']['counter']
        assert_that(
            counter,
            has_entries('C|nginx.http.status.4xx', collected_metric(99),
                        'C|nginx.http.status.discarded', collected_metric(0)))

        # collect single request with $status 499
        tail = [line_template % 499]
        NginxAccessLogsCollector(object=self.fake_object, tail=tail).collect()
        counter = self.fake_object.statsd.flush()['metrics']['counter']
        assert_that(
            counter,
            has_entries('C|nginx.http.status.4xx', collected_metric(1),
                        'C|nginx.http.status.discarded', collected_metric(1)))
Ejemplo n.º 20
0
    def test_empty_upstreams(self):
        log_format = '$remote_addr - $remote_user [$time_local] ' + \
                     '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
                     'rt=$request_time cs=$upstream_cache_status ut="$upstream_response_time"'

        line = \
            '1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \
            '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 cs=- ut="-"'

        # run single method
        collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
        collector.upstreams(NginxAccessLogParser(log_format).parse(line))

        # check
        metrics = self.fake_object.statsd.current
        assert_that(metrics, equal_to(defaultdict()))

        # counters
        counters = metrics['counter']
        assert_that(counters, equal_to({}))

        # histogram
        histogram = metrics['timer']
        assert_that(histogram, equal_to({}))
Ejemplo n.º 21
0
    def test_empty_upstreams(self):
        log_format = '$remote_addr - $remote_user [$time_local] ' + \
                     '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
                     'rt=$request_time cs=$upstream_cache_status ut="$upstream_response_time"'

        line = \
            '1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \
            '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 cs=- ut="-"'

        # run single method
        collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
        collector.upstreams(NginxAccessLogParser(log_format).parse(line))

        # check
        metrics = self.fake_object.statsd.current
        assert_that(metrics, equal_to(defaultdict()))

        # counters
        counters = metrics['counter']
        assert_that(counters, equal_to({}))

        # histogram
        histogram = metrics['timer']
        assert_that(histogram, equal_to({}))
Ejemplo n.º 22
0
    def _setup_access_logs(self):
        # access logs
        for log_description, log_data in self.config.access_logs.iteritems():
            format_name = log_data['log_format']
            log_format = self.config.log_formats.get(format_name)
            tail = self.__setup_pipeline(log_description)

            if tail:
                self.collectors.append(
                    NginxAccessLogsCollector(object=self,
                                             interval=self.intervals['logs'],
                                             log_format=log_format,
                                             tail=tail))

                # Send access log discovery event.
                self.eventd.event(level=INFO,
                                  message='nginx access log %s found' %
                                  log_description)
Ejemplo n.º 23
0
    def test_empty_time_variable(self):
        log_format = ('{'
                      '"remote_addr":"$remote_addr",'
                      '"remote_user":"******",'
                      '"time_local":"$time_local",'
                      '"request_method":"$request_method",'
                      '"request_uri":"$request_uri",'
                      '"status":"$status",'
                      '"bytes_sent":$bytes_sent,'
                      '"http_referer":"$http_referer",'
                      '"http_user_agent":"$http_user_agent",'
                      '"http_cookie":"$http_cookie",'
                      '"sent_http_cookie":"$sent_http_cookie",'
                      '"http_x-forwarded-for":"$http_x_forwarded_for",'
                      '"geoip_city":"",'
                      '"geoip_region":"",'
                      '"geoip_city_country_code":"",'
                      '"http_x_reqid":"$http_x_reqid",'
                      '"request_id":"$request_id",'
                      '"http_cache_control":"$http_cache_control",'
                      '"sent_http_cache_control":"$sent_http_cache_control",'
                      '"host":"$host",'
                      '"sent_http_x_cache_info":"$sent_http_x_cache_info",'
                      '"http_accept_language":"$http_accept_language",'
                      '"sent_http_lf_error":"$sent_http_lf_error",'
                      '"http_device_id":"$http_device_id",'
                      '"http_device_type":"$http_device_type",'
                      '"upstream_connect_time":"$upstream_connect_time",'
                      '"upstream_header_time":"$upstream_header_time",'
                      '"upstream_response_time":"$upstream_response_time",'
                      '"request_time":"$request_time",'
                      '"server_name":"$server_name",'
                      '"http_vsapptype":"$http_vsapptype",'
                      '"http_vsappversion":"$http_vsappversion",'
                      '"http_headers":"$http_headers",'
                      '"sent_http_headers":"$sent_http_headers",'
                      '"upstream_cache_status":"$upstream_cache_status",'
                      '"request_body_masked":"$request_body_masked", '
                      '"http_content_type":"$http_content_type"'
                      '}')

        lines = [(
            '{'
            '"remote_addr":"11.111.11.111",'
            '"remote_user":"",'
            '"time_local":"21/Sep/2018:20:41:39 +0000",'
            '"request_method":"GET",'
            '"request_uri":"/some/request/uri/",'
            '"status":"200",'
            '"bytes_sent":410,'
            '"http_referer":"https://example.com",'
            '"http_user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:62.0) Gecko/20100101 Firefox/62.0",'
            '"http_cookie":"",'
            '"sent_http_cookie":"",'
            '"http_x-forwarded-for":"",'
            '"geoip_city":"",'
            '"geoip_region":"",'
            '"geoip_city_country_code":"",'
            '"http_x_reqid":"",'
            '"request_id":"abcdef0123456789",'
            '"http_cache_control":"",'
            '"sent_http_cache_control":"no-cache",'
            '"host":"example.com",'
            '"sent_http_x_cache_info":"",'
            '"http_accept_language":"en-US,en;q=0.5",'
            '"sent_http_lf_error":"",'
            '"http_device_id":"",'
            '"http_device_type":"",'
            '"upstream_connect_time":"",'
            '"upstream_header_time":"",'
            '"upstream_response_time":"",'
            '"request_time":"0.000",'
            '"server_name":"_",'
            '"http_vsapptype":"",'
            '"http_vsappversion":"",'
            '"http_headers":"",'
            '"sent_http_headers":"",'
            '"upstream_cache_status":"",'
            '"request_body_masked":"", '
            '"http_content_type":""'
            '}')]

        collector = NginxAccessLogsCollector(object=self.fake_object,
                                             log_format=log_format,
                                             tail=lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))
        assert_that(metrics, has_item('timer'))

        counter = metrics['counter']
        timer = metrics['timer']

        assert_that(
            counter,
            has_items(
                'C|nginx.cache.bypass', 'C|nginx.cache.expired',
                'C|nginx.cache.hit', 'C|nginx.cache.miss',
                'C|nginx.cache.revalidated', 'C|nginx.cache.stale',
                'C|nginx.cache.updating', 'C|nginx.http.method.delete',
                'C|nginx.http.method.get', 'C|nginx.http.method.head',
                'C|nginx.http.method.options', 'C|nginx.http.method.other',
                'C|nginx.http.method.post', 'C|nginx.http.method.put',
                'C|nginx.http.request.bytes_sent', 'C|nginx.http.status.1xx',
                'C|nginx.http.status.2xx', 'C|nginx.http.status.3xx',
                'C|nginx.http.status.403', 'C|nginx.http.status.404',
                'C|nginx.http.status.4xx', 'C|nginx.http.status.500',
                'C|nginx.http.status.502', 'C|nginx.http.status.503',
                'C|nginx.http.status.504', 'C|nginx.http.status.5xx',
                'C|nginx.http.status.discarded', 'C|nginx.upstream.next.count',
                'C|nginx.upstream.request.count'))

        assert_that(
            timer,
            has_items('G|nginx.http.request.time',
                      'C|nginx.http.request.time.count',
                      'G|nginx.http.request.time.max',
                      'G|nginx.http.request.time.median',
                      'G|nginx.http.request.time.pctl95'))

        # values
        assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(1))
        assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(1))
        assert_that(counter['C|nginx.http.request.bytes_sent'][0][1],
                    equal_to(410))
        assert_that(timer['C|nginx.http.request.time.count'][0][1],
                    equal_to(1))

        # check zero values
        for counter_name, counter_key in collector.counters.iteritems():
            if counter_key in collector.parser.keys:
                assert_that(counter, has_key('C|%s' % counter_name))
                if counter_name not in ('nginx.http.method.get',
                                        'nginx.http.status.2xx',
                                        'nginx.http.request.bytes_sent'):
                    assert_that(counter['C|%s' % counter_name][0][1],
                                equal_to(0))
            elif counter_key is not None:
                if counter_key not in collector.parser.request_variables:
                    assert_that(counter, not_(has_key('C|%s' % counter_name)))
Ejemplo n.º 24
0
    def test_upstream_status_multi_value(self):
        log_format = '$remote_addr - $remote_user [$time_local] ' + \
                     '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
                     'rt=$request_time us="$upstream_status" ut="$upstream_response_time" cs=$upstream_cache_status'

        lines = [
            '1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' +
            '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 us="200" ut="2.001, 0.345" cs=MISS',

            '1.2.3.4 - - [22/Jan/2010:20:34:21 +0300] "GET /foo/ HTTP/1.1" 300 1078 ' +
            '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 us="502, 504" ut="2.002" cs=HIT',

        ]

        collector = NginxAccessLogsCollector(object=self.fake_object,
                                             log_format=log_format, tail=lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))
        assert_that(metrics, has_item('timer'))

        # counter keys
        counter = metrics['counter']
        for key in ['C|nginx.http.method.get',
                    'C|nginx.http.v1_1',
                    'C|nginx.upstream.next.count',
                    'C|nginx.upstream.request.count',
                    'C|nginx.http.status.3xx',
                    'C|nginx.cache.miss',
                    'C|nginx.http.status.2xx',
                    'C|nginx.http.request.body_bytes_sent',
                    'C|nginx.cache.hit',
                    'C|nginx.upstream.status.2xx',
                    'C|nginx.upstream.status.5xx']:
            assert_that(counter, has_key(key))

        # timer keys
        timer = metrics['timer']
        for key in ['G|nginx.upstream.response.time.pctl95',
                    'C|nginx.upstream.response.time.count',
                    'C|nginx.http.request.time.count',
                    'G|nginx.http.request.time',
                    'G|nginx.http.request.time.pctl95',
                    'G|nginx.http.request.time.median',
                    'G|nginx.http.request.time.max',
                    'G|nginx.upstream.response.time',
                    'G|nginx.upstream.response.time.median',
                    'G|nginx.upstream.response.time.max']:
            assert_that(timer, has_key(key))

        # values
        assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(2))
        assert_that(counter['C|nginx.upstream.request.count'][0][1],
                    equal_to(2))
        assert_that(counter['C|nginx.upstream.status.2xx'][0][1],
                    equal_to(1))
        assert_that(counter['C|nginx.upstream.status.5xx'][0][1],
                    equal_to(2))
        assert_that(counter['C|nginx.upstream.next.count'][0][1], equal_to(1))
        assert_that(timer['G|nginx.upstream.response.time.max'][0][1],
                    equal_to(2.001 + 0.345))

        # check zero values
        for counter_name, counter_key in collector.counters.iteritems():
            if counter_key in collector.parser.keys:
                assert_that(counter, has_key('C|%s' % counter_name))
                if counter_name not in (
                        'nginx.http.status.2xx',
                        'nginx.http.status.3xx',
                        'nginx.http.method.get',
                        'nginx.upstream.request.count',
                        'nginx.upstream.next.count',
                        'nginx.upstream.response.time.max',
                        'nginx.http.v1_1',
                        'nginx.cache.miss',
                        'nginx.cache.hit',
                        'nginx.http.request.body_bytes_sent',
                        'nginx.upstream.status.2xx',
                        'nginx.upstream.status.5xx'
                ):
                    assert_that(counter['C|%s' % counter_name][0][1],
                                equal_to(0))
            elif counter_key is not None:
                if counter_key not in collector.parser.request_variables:
                    assert_that(counter, not_(has_key('C|%s' % counter_name)))
Ejemplo n.º 25
0
    def test_timers_with_filters(self):
        self.fake_object.filters = [
            Filter(filter_rule_id=3,
                   metric='nginx.upstream.response.time.median',
                   data=[['$status', '~', '200']]),
            Filter(filter_rule_id=4,
                   metric='nginx.upstream.response.time.max',
                   data=[['$status', '~', '400']])
        ]

        collector = NginxAccessLogsCollector(
            object=self.fake_object,
            log_format=
            '$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent '
            +
            '\"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\" '
            +
            'rt=$request_time ua=\"$upstream_addr\" us=\"$upstream_status\" ' +
            'ut=\"$upstream_response_time\" ul=\"$upstream_response_length\" '
            + 'cs=$upstream_cache_status sn=$server_name',
            tail=[
                '104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 200 28275 "-" '
                +
                '"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="1.000" ul="-" cs=- sn=differentsimgirls.com',
                '104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 200 28275 "-" '
                +
                '"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="3.000" ul="-" cs=- sn=differentsimgirls.com',
                '104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 200 28275 "-" '
                +
                '"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="5.000" ul="-" cs=- sn=differentsimgirls.com',
                '104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 400 28275 "-" '
                +
                '"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="7.000" ul="-" cs=- sn=differentsimgirls.com'
            ])
        collector.collect()

        metrics = self.fake_object.statsd.flush()['metrics']
        timer = metrics['timer']
        for key in [
                'G|nginx.upstream.response.time.max',
                'G|nginx.upstream.response.time.median',
                'G|nginx.upstream.response.time.pctl95',
                'C|nginx.upstream.response.time.count',
                'G|nginx.upstream.response.time.max||3',
                'G|nginx.upstream.response.time.median||3',
                'G|nginx.upstream.response.time.pctl95||3',
                'C|nginx.upstream.response.time.count||3',
                'G|nginx.upstream.response.time.max||4',
                'G|nginx.upstream.response.time.median||4',
                'G|nginx.upstream.response.time.pctl95||4',
                'C|nginx.upstream.response.time.count||4'
        ]:
            assert_that(timer, has_key(key))

        assert_that(timer["G|nginx.upstream.response.time.max||3"][0][1],
                    equal_to(5.000))
        assert_that(timer["G|nginx.upstream.response.time.median||3"][0][1],
                    equal_to(3.000))
        assert_that(timer["C|nginx.upstream.response.time.count||3"][0][1],
                    equal_to(3))
        assert_that(timer["G|nginx.upstream.response.time.max||4"][0][1],
                    equal_to(7.000))
        assert_that(timer["G|nginx.upstream.response.time.median||4"][0][1],
                    equal_to(7.000))
        assert_that(timer["C|nginx.upstream.response.time.count||4"][0][1],
                    equal_to(1))
        assert_that(timer["G|nginx.upstream.response.time.max"][0][1],
                    equal_to(7.000))
        assert_that(timer["G|nginx.upstream.response.time.median"][0][1],
                    equal_to(4.000))
        assert_that(timer["C|nginx.upstream.response.time.count"][0][1],
                    equal_to(4))
Ejemplo n.º 26
0
    def test_extend_duplicates_reported(self):
        """
        Test the specific reported bug format.
        """
        log_format = '$remote_addr - $remote_user [$time_local]  ' + \
                     '"$request" $status $body_bytes_sent ' + \
                     '"$http_referer" "$http_user_agent" ' + \
                     '$request_length $body_bytes_sent'

        lines = [
            '188.165.1.1 - - [17/Nov/2015:22:07:42 +0100]  ' +
            '"GET /2014/09/quicktipp-phpmyadmin-update-script/?pk_campaign=feed&pk_kwd=quicktipp-' +
            'phpmyadmin-update-script HTTP/1.1" ' +
            '200 41110 "http://www.google.co.uk/url?sa=t&source=web&cd=1" "Mozilla/5.0 (Windows NT 6.1; WOW64) ' +
            'AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.92 Safari/535.2" 327 41110',

            '192.168.100.200 - - [17/Nov/2015:22:09:26 +0100]  "POST /wp-cron.php?doing_wp_cron=1447794566.' +
            '5160338878631591796875 HTTP/1.0" 200 0 "-" "WordPress/4.3.1; http://my.domain.at.private.com" 281 0'
        ]

        collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))

        # counter keys
        counter = metrics['counter']
        for key in ['C|nginx.http.method.post', 'C|nginx.http.method.get', 'C|nginx.http.status.2xx',
                    'C|nginx.http.v1_1', 'C|nginx.http.request.body_bytes_sent',
                    'C|nginx.http.v1_0']:
            assert_that(counter, has_key(key))

        # average keys
        averages = metrics['average']
        assert_that(averages, has_key('G|nginx.http.request.length'))

        # values
        assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(1))
        assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(1))
        assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2))
        assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1], equal_to(41110))
        assert_that(averages['G|nginx.http.request.length'][0][1], equal_to((327+281)/2))

        # check zero values
        for counter_name, counter_key in collector.counters.iteritems():
            if counter_key in collector.parser.keys:
                assert_that(counter, has_key('C|%s' % counter_name))
                if counter_name not in (
                    'nginx.http.status.2xx',
                    'nginx.http.method.post',
                    'nginx.http.method.get',
                    'nginx.http.request.length',
                    'nginx.http.request.body_bytes_sent',
                    'nginx.http.v1_1',
                    'nginx.http.v1_0'
                ):
                    assert_that(counter['C|%s' % counter_name][0][1], equal_to(0))
            elif counter_key is not None:
                if counter_key not in collector.parser.request_variables:
                    assert_that(counter, not_(has_key('C|%s' % counter_name)))
Ejemplo n.º 27
0
    def test_extend_duplicates_reported(self):
        """
        Test the specific reported bug format.
        """
        log_format = '$remote_addr - $remote_user [$time_local]  ' + \
                     '"$request" $status $body_bytes_sent ' + \
                     '"$http_referer" "$http_user_agent" ' + \
                     '$request_length $body_bytes_sent'

        lines = [
            '188.165.1.1 - - [17/Nov/2015:22:07:42 +0100]  ' +
            '"GET /2014/09/quicktipp-phpmyadmin-update-script/?pk_campaign=feed&pk_kwd=quicktipp-' +
            'phpmyadmin-update-script HTTP/1.1" ' +
            '200 41110 "http://www.google.co.uk/url?sa=t&source=web&cd=1" "Mozilla/5.0 (Windows NT 6.1; WOW64) ' +
            'AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.92 Safari/535.2" 327 41110',

            '192.168.100.200 - - [17/Nov/2015:22:09:26 +0100]  "POST /wp-cron.php?doing_wp_cron=1447794566.' +
            '5160338878631591796875 HTTP/1.0" 200 0 "-" "WordPress/4.3.1; http://my.domain.at.private.com" 281 0'
        ]

        collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))

        # counter keys
        counter = metrics['counter']
        for key in ['C|nginx.http.method.post', 'C|nginx.http.method.get', 'C|nginx.http.status.2xx',
                    'C|nginx.http.v1_1', 'C|nginx.http.request.body_bytes_sent',
                    'C|nginx.http.v1_0']:
            assert_that(counter, has_key(key))

        # average keys
        averages = metrics['average']
        assert_that(averages, has_key('G|nginx.http.request.length'))

        # values
        assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(1))
        assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(1))
        assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2))
        assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1], equal_to(41110))
        assert_that(averages['G|nginx.http.request.length'][0][1], equal_to((327+281)/2))

        # check zero values
        for counter_name, counter_key in collector.counters.iteritems():
            if counter_key in collector.parser.keys:
                assert_that(counter, has_key('C|%s' % counter_name))
                if counter_name not in (
                    'nginx.http.status.2xx',
                    'nginx.http.method.post',
                    'nginx.http.method.get',
                    'nginx.http.request.length',
                    'nginx.http.request.body_bytes_sent',
                    'nginx.http.v1_1',
                    'nginx.http.v1_0'
                ):
                    assert_that(counter['C|%s' % counter_name][0][1], equal_to(0))
            else:
                if counter_key not in collector.parser.request_variables:
                    assert_that(counter, not_(has_key('C|%s' % counter_name)))
Ejemplo n.º 28
0
    def test_extend_duplicates(self):
        """
        Test a log format that defines duplicate variables.
        """
        log_format = '$remote_addr - $remote_addr - $remote_addr - $remote_user [$time_local] ' + \
                     '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
                     'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status'

        lines = [
            '1.2.3.4 - 1.2.3.4 - 1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' +
            '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS',

            '1.2.3.4 - 1.2.3.4 - 1.2.3.4 - - [22/Jan/2010:20:34:21 +0300] "GET /foo/ HTTP/1.1" 300 1078 ' +
            '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.002" cs=HIT',

        ]

        collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines)
        collector.collect()

        # check
        metrics = self.fake_object.statsd.flush()['metrics']
        assert_that(metrics, has_item('counter'))
        assert_that(metrics, has_item('timer'))

        # counter keys
        counter = metrics['counter']
        for key in ['C|nginx.http.method.get', 'C|nginx.http.v1_1', 'C|nginx.upstream.next.count',
                    'C|nginx.upstream.request.count', 'C|nginx.http.status.3xx', 'C|nginx.cache.miss',
                    'C|nginx.http.status.2xx', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.cache.hit']:
            assert_that(counter, has_key(key))

        # timer keys
        timer = metrics['timer']
        for key in ['G|nginx.upstream.response.time.pctl95', 'C|nginx.upstream.response.time.count',
                    'C|nginx.http.request.time.count', 'G|nginx.http.request.time',
                    'G|nginx.http.request.time.pctl95', 'G|nginx.http.request.time.median',
                    'G|nginx.http.request.time.max', 'G|nginx.upstream.response.time',
                    'G|nginx.upstream.response.time.median', 'G|nginx.upstream.response.time.max']:
            assert_that(timer, has_key(key))

        # values
        assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(2))
        assert_that(counter['C|nginx.upstream.request.count'][0][1], equal_to(2))
        assert_that(counter['C|nginx.upstream.next.count'][0][1], equal_to(1))
        assert_that(timer['G|nginx.upstream.response.time.max'][0][1], equal_to(2.001+0.345))

        # check zero values
        for counter_name, counter_key in collector.counters.iteritems():
            if counter_key in collector.parser.keys:
                assert_that(counter, has_key('C|%s' % counter_name))
                if counter_name not in (
                    'nginx.http.status.2xx',
                    'nginx.http.status.3xx',
                    'nginx.http.method.get',
                    'nginx.upstream.request.count',
                    'nginx.upstream.next.count',
                    'nginx.upstream.response.time.max',
                    'nginx.http.v1_1',
                    'nginx.cache.miss',
                    'nginx.cache.hit',
                    'nginx.http.request.body_bytes_sent'
                ):
                    assert_that(counter['C|%s' % counter_name][0][1], equal_to(0))
            else:
                if counter_key not in collector.parser.request_variables:
                    assert_that(counter, not_(has_key('C|%s' % counter_name)))