def test_server_name(self): self.fake_object.filters = [ Filter(**dict(filter_rule_id=2, metric='nginx.http.status.2xx', data=[{ u'$server_name': u'differentsimgirls.com' }])) ] collector = NginxAccessLogsCollector( object=self.fake_object, log_format= '$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent ' + '\"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\" ' + 'rt=$request_time ua=\"$upstream_addr\" us=\"$upstream_status\" ' + 'ut=\"$upstream_response_time\" ul=\"$upstream_response_length\" ' + 'cs=$upstream_cache_status sn=$server_name', tail=[ '104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 200 28275 "-" ' + '"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="-" ul="-" cs=- sn=differentsimgirls.com' ]) collector.collect() # check metrics = self.fake_object.statsd.flush()['metrics'] assert_that(metrics, has_item('counter')) counter = metrics['counter'] # check our metric assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(1)) assert_that(counter['C|nginx.http.status.2xx||2'][0][1], equal_to(1))
def test_server_name(self): self.fake_object.filters = [ Filter(**dict( filter_rule_id=2, metric='nginx.http.status.2xx', data=[ {u'$server_name': u'differentsimgirls.com'} ] )) ] collector = NginxAccessLogsCollector( object=self.fake_object, log_format='$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent ' + '\"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\" ' + 'rt=$request_time ua=\"$upstream_addr\" us=\"$upstream_status\" ' + 'ut=\"$upstream_response_time\" ul=\"$upstream_response_length\" ' + 'cs=$upstream_cache_status sn=$server_name', tail=[ '104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 200 28275 "-" ' + '"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="-" ul="-" cs=- sn=differentsimgirls.com' ] ) collector.collect() # check metrics = self.fake_object.statsd.flush()['metrics'] assert_that(metrics, has_item('counter')) counter = metrics['counter'] # check our metric assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(1)) assert_that(counter['C|nginx.http.status.2xx||2'][0][1], equal_to(1))
def test_upstreams(self): log_format = '$remote_addr - $remote_user [$time_local] ' + \ '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \ 'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status' line = \ '1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \ '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS' # run single method collector = NginxAccessLogsCollector(object=self.fake_object, tail=[]) collector.upstreams(NginxAccessLogParser(log_format).parse(line)) # check metrics = self.fake_object.statsd.current assert_that(metrics, has_item('counter')) assert_that(metrics, has_item('timer')) # counters counters = metrics['counter'] assert_that(counters, has_item('nginx.upstream.request.count')) assert_that(counters, has_item('nginx.upstream.next.count')) assert_that(counters, has_item('nginx.cache.miss')) assert_that(counters['nginx.upstream.request.count'][0][1], equal_to(1)) assert_that(counters['nginx.upstream.next.count'][0][1], equal_to(1)) assert_that(counters['nginx.cache.miss'][0][1], equal_to(1)) # histogram histogram = metrics['timer'] assert_that(histogram, has_item('nginx.upstream.response.time')) assert_that(histogram['nginx.upstream.response.time'], equal_to([2.001 + 0.345]))
def test_upstreams(self): log_format = '$remote_addr - $remote_user [$time_local] ' + \ '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \ 'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status' line = \ '1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \ '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS' # run single method collector = NginxAccessLogsCollector(object=self.fake_object, tail=[]) collector.upstreams(NginxAccessLogParser(log_format).parse(line)) # check metrics = self.fake_object.statsd.current assert_that(metrics, has_item('counter')) assert_that(metrics, has_item('timer')) # counters counters = metrics['counter'] assert_that(counters, has_item('nginx.upstream.request.count')) assert_that(counters, has_item('nginx.upstream.next.count')) assert_that(counters, has_item('nginx.cache.miss')) assert_that(counters['nginx.upstream.request.count'][0][1], equal_to(1)) assert_that(counters['nginx.upstream.next.count'][0][1], equal_to(1)) assert_that(counters['nginx.cache.miss'][0][1], equal_to(1)) # histogram histogram = metrics['timer'] assert_that(histogram, has_item('nginx.upstream.response.time')) assert_that(histogram['nginx.upstream.response.time'], equal_to([2.001 + 0.345]))
def test_regex_filter(self): self.fake_object.filters = [ Filter(**dict( filter_rule_id=2, metric='nginx.http.request.body_bytes_sent', data=[ {'$request_uri': '/img.*'}, {'$server_protocol': 'HTTP/1.2'} ] )) ] collector = NginxAccessLogsCollector(object=self.fake_object, tail=self.lines) collector.collect() # check metrics = self.fake_object.statsd.flush()['metrics'] assert_that(metrics, has_item('counter')) # counters counter = metrics['counter'] for key in ('C|nginx.http.method.get', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.status.3xx', 'C|nginx.http.status.2xx', 'C|nginx.http.method.post', 'C|nginx.http.v1_1', 'C|nginx.http.status.4xx', 'C|nginx.http.request.body_bytes_sent||2', 'C|nginx.http.method.post'): assert_that(counter, has_key(key)) # values assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(4)) assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(2)) assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(3)) # filter values assert_that(counter['C|nginx.http.request.body_bytes_sent||2'][0][1], equal_to(2))
def test_combined(self): lines = [ '178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /img/docker.png HTTP/1.1" 304 0 ' + '"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' + '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/43.0.2357.124 Safari/537.36"', '178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /api/inventory/objects/ HTTP/1.1" 200 1093 ' + '"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' + '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/43.0.2357.124 Safari/537.36"', '127.0.0.1 - - [18/Jun/2015:17:22:33 +0000] "POST /1.0/589fjinijenfirjf/meta/ HTTP/1.1" ' + '202 2 "-" "python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"', '52.6.158.18 - - [18/Jun/2015:17:22:40 +0000] "GET /#/objects HTTP/1.1" 416 84 ' + '"-" "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)"' ] collector = NginxAccessLogsCollector(object=self.fake_object, tail=lines) collector.collect() # check metrics = self.fake_object.statsd.flush()['metrics'] assert_that(metrics, has_item('counter')) # counters counter = metrics['counter'] for key in ('C|nginx.http.method.get', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.status.3xx', 'C|nginx.http.status.2xx', 'C|nginx.http.method.post', 'C|nginx.http.v1_1', 'C|nginx.http.status.4xx'): assert_that(counter, has_key(key)) # values assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(3)) assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2)) assert_that(counter['C|nginx.http.v1_1'][0][1], equal_to(4)) assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1], equal_to(84 + 2 + 1093 + 0)) # check zero values for counter_name, counter_key in collector.counters.iteritems(): if counter_key in collector.parser.keys: assert_that(counter, has_key('C|%s' % counter_name)) if counter_name not in ('nginx.http.status.2xx', 'nginx.http.status.3xx', 'nginx.http.status.4xx', 'nginx.http.v1_1', 'nginx.http.request.body_bytes_sent'): assert_that(counter['C|%s' % counter_name][0][1], equal_to(0)) else: if counter_key not in collector.parser.request_variables: assert_that(counter, not_(has_key('C|%s' % counter_name)))
def test_combined(self): lines = [ '178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /img/docker.png HTTP/1.1" 304 0 ' + '"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' + '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/43.0.2357.124 Safari/537.36"', '178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /api/inventory/objects/ HTTP/1.1" 200 1093 ' + '"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' + '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/43.0.2357.124 Safari/537.36"', '127.0.0.1 - - [18/Jun/2015:17:22:33 +0000] "POST /1.0/589fjinijenfirjf/meta/ HTTP/1.1" ' + '202 2 "-" "python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"', '52.6.158.18 - - [18/Jun/2015:17:22:40 +0000] "GET /#/objects HTTP/1.1" 416 84 ' + '"-" "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)"' ] collector = NginxAccessLogsCollector(object=self.fake_object, tail=lines) collector.collect() # check metrics = self.fake_object.statsd.flush()['metrics'] assert_that(metrics, has_item('counter')) # counters counter = metrics['counter'] for key in ('C|nginx.http.method.get', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.status.3xx', 'C|nginx.http.status.2xx','C|nginx.http.method.post', 'C|nginx.http.v1_1', 'C|nginx.http.status.4xx'): assert_that(counter, has_key(key)) # values assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(3)) assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2)) assert_that(counter['C|nginx.http.v1_1'][0][1], equal_to(4)) assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1], equal_to(84 + 2 + 1093 + 0)) # check zero values for counter_name, counter_key in collector.counters.iteritems(): if counter_key in collector.parser.keys: assert_that(counter, has_key('C|%s' % counter_name)) if counter_name not in ( 'nginx.http.status.2xx', 'nginx.http.status.3xx', 'nginx.http.status.4xx', 'nginx.http.v1_1', 'nginx.http.request.body_bytes_sent' ): assert_that(counter['C|%s' % counter_name][0][1], equal_to(0)) else: if counter_key not in collector.parser.request_variables: assert_that(counter, not_(has_key('C|%s' % counter_name)))
def test_http_status(self): line = '127.0.0.1 - - [02/Jul/2015:14:49:48 +0000] "GET /basic_status HTTP/1.1" 200 110 "-" ' + \ '"python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"' # run single method collector = NginxAccessLogsCollector(object=self.fake_object, tail=[]) collector.http_status(NginxAccessLogParser().parse(line)) # check metrics = self.fake_object.statsd.current assert_that(metrics, has_item('counter')) counters = metrics['counter'] assert_that(counters, has_item('nginx.http.status.2xx')) assert_that(counters['nginx.http.status.2xx'][0][1], equal_to(1))
def test_http_status(self): line = '127.0.0.1 - - [02/Jul/2015:14:49:48 +0000] "GET /basic_status HTTP/1.1" 200 110 "-" ' + \ '"python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"' # run single method collector = NginxAccessLogsCollector(object=self.fake_object, tail=[]) collector.http_status(NginxAccessLogParser().parse(line)) # check metrics = self.fake_object.statsd.current assert_that(metrics, has_item('counter')) counters = metrics['counter'] assert_that(counters, has_item('nginx.http.status.2xx')) assert_that(counters['nginx.http.status.2xx'][0][1], equal_to(1))
def test_simple_filter(self): self.fake_object.filters = [ Filter(**dict(filter_rule_id=1, metric='nginx.http.status.2xx', data=[{ '$request_method': 'GET' }, { '$status': '200' }])), Filter(**dict(filter_rule_id=2, metric='nginx.http.request.body_bytes_sent', data=[{ '$request_uri': '/img*' }, { '$server_protocol': 'HTTP/1.2' }])) ] collector = NginxAccessLogsCollector(object=self.fake_object, tail=self.lines) collector.collect() # check metrics = self.fake_object.statsd.flush()['metrics'] assert_that(metrics, has_item('counter')) # counters counter = metrics['counter'] for key in ('C|nginx.http.method.get', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.status.3xx', 'C|nginx.http.status.2xx', 'C|nginx.http.method.post', 'C|nginx.http.v1_1', 'C|nginx.http.status.4xx', 'C|nginx.http.status.2xx||1', 'C|nginx.http.method.post'): assert_that(counter, has_key(key)) # values assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(4)) assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(2)) assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(3)) # filter values assert_that(counter['C|nginx.http.status.2xx||1'][0][1], equal_to(2))
def test_empty_upstreams(self): log_format = '$remote_addr - $remote_user [$time_local] ' + \ '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \ 'rt=$request_time cs=$upstream_cache_status ut="$upstream_response_time"' line = \ '1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \ '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 cs=- ut="-"' # run single method collector = NginxAccessLogsCollector(object=self.fake_object, tail=[]) collector.upstreams(NginxAccessLogParser(log_format).parse(line)) # check metrics = self.fake_object.statsd.current assert_that(metrics, equal_to(defaultdict())) # counters counters = metrics['counter'] assert_that(counters, equal_to({})) # histogram histogram = metrics['timer'] assert_that(histogram, equal_to({}))
def test_empty_upstreams(self): log_format = '$remote_addr - $remote_user [$time_local] ' + \ '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \ 'rt=$request_time cs=$upstream_cache_status ut="$upstream_response_time"' line = \ '1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \ '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 cs=- ut="-"' # run single method collector = NginxAccessLogsCollector(object=self.fake_object, tail=[]) collector.upstreams(NginxAccessLogParser(log_format).parse(line)) # check metrics = self.fake_object.statsd.current assert_that(metrics, equal_to(defaultdict())) # counters counters = metrics['counter'] assert_that(counters, equal_to({})) # histogram histogram = metrics['timer'] assert_that(histogram, equal_to({}))
def __init__(self, **kwargs): super(NginxObject, self).__init__(**kwargs) self.root_uuid = self.data.get('root_uuid') or context.objects.root_object.uuid if context.objects.root_object else None self.local_id_cache = self.data['local_id'] # Assigned by manager self.pid = self.data['pid'] self.version = self.data['version'] self.workers = self.data['workers'] self.prefix = self.data['prefix'] self.bin_path = self.data['bin_path'] self.conf_path = self.data['conf_path'] default_config = context.app_config['containers'][self.type] self.upload_config = self.data.get('upload_config') or default_config.get('upload_config', False) self.run_config_test = self.data.get('run_test') or default_config.get('run_test', False) self.upload_ssl = self.data.get('upload_ssl') or default_config.get('upload_ssl', False) self.config = NginxConfig(self.conf_path, prefix=self.prefix) self.config.full_parse() self.plus_status_external_url, self.plus_status_internal_url = self.get_alive_plus_status_urls() self.plus_status_enabled = True if (self.plus_status_external_url or self.plus_status_internal_url) else False self.stub_status_url = self.get_alive_stub_status_url() self.stub_status_enabled = True if self.stub_status_url else False self.processes = [] self.filters = [] # filters for raw_filter in self.data.get('filters') or []: self.filters.append(Filter(**raw_filter)) # meta meta_collector_class = NginxCommonMetaCollector if host.os_name() == 'linux': if host.linux_name() in ('ubuntu', 'debian'): meta_collector_class = NginxDebianMetaCollector elif host.linux_name() in ('centos',): meta_collector_class = NginxCentosMetaCollector self.collectors = [ meta_collector_class( object=self, interval=self.intervals['meta'] ), NginxMetricsCollector( object=self, interval=self.intervals['metrics'] ), NginxConfigCollector( object=self, interval=self.intervals['configs'], ) ] # access logs for log_filename, format_name in self.config.access_logs.iteritems(): log_format = self.config.log_formats.get(format_name) try: self.collectors.append( NginxAccessLogsCollector( object=self, interval=self.intervals['logs'], filename=log_filename, log_format=log_format, ) ) # Send access log discovery event. self.eventd.event(level=INFO, message='nginx access log %s found' % log_filename) except IOError as e: exception_name = e.__class__.__name__ context.log.error( 'failed to start reading log %s due to %s (maybe has no rights?)' % (log_filename, exception_name) ) context.log.debug('additional info:', exc_info=True) # error logs for log_filename, log_level in self.config.error_logs.iteritems(): try: self.collectors.append( NginxErrorLogsCollector( object=self, interval=self.intervals['logs'], filename=log_filename, level=log_level ) ) # Send error log discovery event. self.eventd.event(level=INFO, message='nginx error log %s found' % log_filename) except IOError as e: exception_name = e.__class__.__name__ context.log.error( 'failed to start reading log %s due to %s (maybe has no rights?)' % (log_filename, exception_name) ) context.log.debug('additional info:', exc_info=True)
def test_extended(self): log_format = '$remote_addr - $remote_user [$time_local] ' + \ '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \ 'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status' lines = [ '1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS', '1.2.3.4 - - [22/Jan/2010:20:34:21 +0300] "GET /foo/ HTTP/1.1" 300 1078 ' + '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.002" cs=HIT', ] collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines) collector.collect() # check metrics = self.fake_object.statsd.flush()['metrics'] assert_that(metrics, has_item('counter')) assert_that(metrics, has_item('timer')) # counter keys counter = metrics['counter'] for key in [ 'C|nginx.http.method.get', 'C|nginx.http.v1_1', 'C|nginx.upstream.next.count', 'C|nginx.upstream.request.count', 'C|nginx.http.status.3xx', 'C|nginx.cache.miss', 'C|nginx.http.status.2xx', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.cache.hit' ]: assert_that(counter, has_key(key)) # timer keys timer = metrics['timer'] for key in [ 'G|nginx.upstream.response.time.pctl95', 'C|nginx.upstream.response.time.count', 'C|nginx.http.request.time.count', 'G|nginx.http.request.time', 'G|nginx.http.request.time.pctl95', 'G|nginx.http.request.time.median', 'G|nginx.http.request.time.max', 'G|nginx.upstream.response.time', 'G|nginx.upstream.response.time.median', 'G|nginx.upstream.response.time.max' ]: assert_that(timer, has_key(key)) # values assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(2)) assert_that(counter['C|nginx.upstream.request.count'][0][1], equal_to(2)) assert_that(counter['C|nginx.upstream.next.count'][0][1], equal_to(1)) assert_that(timer['G|nginx.upstream.response.time.max'][0][1], equal_to(2.001 + 0.345)) # check zero values for counter_name, counter_key in collector.counters.iteritems(): if counter_key in collector.parser.keys: assert_that(counter, has_key('C|%s' % counter_name)) if counter_name not in ('nginx.http.status.2xx', 'nginx.http.status.3xx', 'nginx.http.method.get', 'nginx.upstream.request.count', 'nginx.upstream.next.count', 'nginx.upstream.response.time.max', 'nginx.http.v1_1', 'nginx.cache.miss', 'nginx.cache.hit', 'nginx.http.request.body_bytes_sent'): assert_that(counter['C|%s' % counter_name][0][1], equal_to(0)) else: if counter_key not in collector.parser.request_variables: assert_that(counter, not_(has_key('C|%s' % counter_name)))
def test_extend_duplicates_reported(self): """ Test the specific reported bug format. """ log_format = '$remote_addr - $remote_user [$time_local] ' + \ '"$request" $status $body_bytes_sent ' + \ '"$http_referer" "$http_user_agent" ' + \ '$request_length $body_bytes_sent' lines = [ '188.165.1.1 - - [17/Nov/2015:22:07:42 +0100] ' + '"GET /2014/09/quicktipp-phpmyadmin-update-script/?pk_campaign=feed&pk_kwd=quicktipp-' + 'phpmyadmin-update-script HTTP/1.1" ' + '200 41110 "http://www.google.co.uk/url?sa=t&source=web&cd=1" "Mozilla/5.0 (Windows NT 6.1; WOW64) ' + 'AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.92 Safari/535.2" 327 41110', '192.168.100.200 - - [17/Nov/2015:22:09:26 +0100] "POST /wp-cron.php?doing_wp_cron=1447794566.' + '5160338878631591796875 HTTP/1.0" 200 0 "-" "WordPress/4.3.1; http://my.domain.at.private.com" 281 0' ] collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines) collector.collect() # check metrics = self.fake_object.statsd.flush()['metrics'] assert_that(metrics, has_item('counter')) # counter keys counter = metrics['counter'] for key in [ 'C|nginx.http.method.post', 'C|nginx.http.method.get', 'C|nginx.http.status.2xx', 'C|nginx.http.v1_1', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.v1_0' ]: assert_that(counter, has_key(key)) # average keys averages = metrics['average'] assert_that(averages, has_key('G|nginx.http.request.length')) # values assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(1)) assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(1)) assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2)) assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1], equal_to(41110)) assert_that(averages['G|nginx.http.request.length'][0][1], equal_to((327 + 281) / 2)) # check zero values for counter_name, counter_key in collector.counters.iteritems(): if counter_key in collector.parser.keys: assert_that(counter, has_key('C|%s' % counter_name)) if counter_name not in ('nginx.http.status.2xx', 'nginx.http.method.post', 'nginx.http.method.get', 'nginx.http.request.length', 'nginx.http.request.body_bytes_sent', 'nginx.http.v1_1', 'nginx.http.v1_0'): assert_that(counter['C|%s' % counter_name][0][1], equal_to(0)) else: if counter_key not in collector.parser.request_variables: assert_that(counter, not_(has_key('C|%s' % counter_name)))
def test_extended(self): log_format = '$remote_addr - $remote_user [$time_local] ' + \ '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \ 'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status' lines = [ '1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS', '1.2.3.4 - - [22/Jan/2010:20:34:21 +0300] "GET /foo/ HTTP/1.1" 300 1078 ' + '"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.002" cs=HIT', ] collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines) collector.collect() # check metrics = self.fake_object.statsd.flush()['metrics'] assert_that(metrics, has_item('counter')) assert_that(metrics, has_item('timer')) # counter keys counter = metrics['counter'] for key in ['C|nginx.http.method.get', 'C|nginx.http.v1_1', 'C|nginx.upstream.next.count', 'C|nginx.upstream.request.count', 'C|nginx.http.status.3xx', 'C|nginx.cache.miss', 'C|nginx.http.status.2xx', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.cache.hit']: assert_that(counter, has_key(key)) # timer keys timer = metrics['timer'] for key in ['G|nginx.upstream.response.time.pctl95', 'C|nginx.upstream.response.time.count', 'C|nginx.http.request.time.count', 'G|nginx.http.request.time', 'G|nginx.http.request.time.pctl95', 'G|nginx.http.request.time.median', 'G|nginx.http.request.time.max', 'G|nginx.upstream.response.time', 'G|nginx.upstream.response.time.median', 'G|nginx.upstream.response.time.max']: assert_that(timer, has_key(key)) # values assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(2)) assert_that(counter['C|nginx.upstream.request.count'][0][1], equal_to(2)) assert_that(counter['C|nginx.upstream.next.count'][0][1], equal_to(1)) assert_that(timer['G|nginx.upstream.response.time.max'][0][1], equal_to(2.001+0.345)) # check zero values for counter_name, counter_key in collector.counters.iteritems(): if counter_key in collector.parser.keys: assert_that(counter, has_key('C|%s' % counter_name)) if counter_name not in ( 'nginx.http.status.2xx', 'nginx.http.status.3xx', 'nginx.http.method.get', 'nginx.upstream.request.count', 'nginx.upstream.next.count', 'nginx.upstream.response.time.max', 'nginx.http.v1_1', 'nginx.cache.miss', 'nginx.cache.hit', 'nginx.http.request.body_bytes_sent' ): assert_that(counter['C|%s' % counter_name][0][1], equal_to(0)) else: if counter_key not in collector.parser.request_variables: assert_that(counter, not_(has_key('C|%s' % counter_name)))
def test_extend_duplicates_reported(self): """ Test the specific reported bug format. """ log_format = '$remote_addr - $remote_user [$time_local] ' + \ '"$request" $status $body_bytes_sent ' + \ '"$http_referer" "$http_user_agent" ' + \ '$request_length $body_bytes_sent' lines = [ '188.165.1.1 - - [17/Nov/2015:22:07:42 +0100] ' + '"GET /2014/09/quicktipp-phpmyadmin-update-script/?pk_campaign=feed&pk_kwd=quicktipp-' + 'phpmyadmin-update-script HTTP/1.1" ' + '200 41110 "http://www.google.co.uk/url?sa=t&source=web&cd=1" "Mozilla/5.0 (Windows NT 6.1; WOW64) ' + 'AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.92 Safari/535.2" 327 41110', '192.168.100.200 - - [17/Nov/2015:22:09:26 +0100] "POST /wp-cron.php?doing_wp_cron=1447794566.' + '5160338878631591796875 HTTP/1.0" 200 0 "-" "WordPress/4.3.1; http://my.domain.at.private.com" 281 0' ] collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines) collector.collect() # check metrics = self.fake_object.statsd.flush()['metrics'] assert_that(metrics, has_item('counter')) # counter keys counter = metrics['counter'] for key in ['C|nginx.http.method.post', 'C|nginx.http.method.get', 'C|nginx.http.status.2xx', 'C|nginx.http.v1_1', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.v1_0']: assert_that(counter, has_key(key)) # average keys averages = metrics['average'] assert_that(averages, has_key('G|nginx.http.request.length')) # values assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(1)) assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(1)) assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2)) assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1], equal_to(41110)) assert_that(averages['G|nginx.http.request.length'][0][1], equal_to((327+281)/2)) # check zero values for counter_name, counter_key in collector.counters.iteritems(): if counter_key in collector.parser.keys: assert_that(counter, has_key('C|%s' % counter_name)) if counter_name not in ( 'nginx.http.status.2xx', 'nginx.http.method.post', 'nginx.http.method.get', 'nginx.http.request.length', 'nginx.http.request.body_bytes_sent', 'nginx.http.v1_1', 'nginx.http.v1_0' ): assert_that(counter['C|%s' % counter_name][0][1], equal_to(0)) else: if counter_key not in collector.parser.request_variables: assert_that(counter, not_(has_key('C|%s' % counter_name)))