def test_upstreams(self):
log_format = '$remote_addr - $remote_user [$time_local] ' + \
'"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status'
line = \
'1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \
'"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS'
# run single method
collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
collector.upstreams(NginxAccessLogParser(log_format).parse(line))
# check
metrics = self.fake_object.statsd.current
assert_that(metrics, has_item('counter'))
assert_that(metrics, has_item('timer'))
# counters
counters = metrics['counter']
assert_that(counters, has_item('nginx.upstream.request.count'))
assert_that(counters, has_item('nginx.upstream.next.count'))
assert_that(counters, has_item('nginx.cache.miss'))
assert_that(counters['nginx.upstream.request.count'][0][1], equal_to(1))
assert_that(counters['nginx.upstream.next.count'][0][1], equal_to(1))
assert_that(counters['nginx.cache.miss'][0][1], equal_to(1))
# histogram
histogram = metrics['timer']
assert_that(histogram, has_item('nginx.upstream.response.time'))
assert_that(histogram['nginx.upstream.response.time'], equal_to([2.001 + 0.345]))
def test_regex_filter(self):
self.fake_object.filters = [
Filter(**dict(
filter_rule_id=2,
metric='nginx.http.request.body_bytes_sent',
data=[
{'$request_uri': '/img*'},
{'$server_protocol': 'HTTP/1.2'}
]
))
]
collector = NginxAccessLogsCollector(object=self.fake_object, tail=self.lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counters
counter = metrics['counter']
for key in ('C|nginx.http.method.get', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.status.3xx',
'C|nginx.http.status.2xx', 'C|nginx.http.method.post', 'C|nginx.http.v1_1',
'C|nginx.http.status.4xx', 'C|nginx.http.request.body_bytes_sent||2', 'C|nginx.http.method.post'):
assert_that(counter, has_key(key))
# values
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(4))
assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(2))
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(3))
# filter values
assert_that(counter['C|nginx.http.request.body_bytes_sent||2'][0][1], equal_to(2))
def test_extend_duplicates_reported(self):
"""
Test the specific reported bug format.
"""
log_format = '$remote_addr - $remote_user [$time_local] ' + \
'"$request" $status $body_bytes_sent ' + \
'"$http_referer" "$http_user_agent" ' + \
'$request_length $body_bytes_sent'
lines = [
'188.165.1.1 - - [17/Nov/2015:22:07:42 +0100] ' +
'"GET /2014/09/quicktipp-phpmyadmin-update-script/?pk_campaign=feed&pk_kwd=quicktipp-' +
'phpmyadmin-update-script HTTP/1.1" ' +
'200 41110 "http://www.google.co.uk/url?sa=t&source=web&cd=1" "Mozilla/5.0 (Windows NT 6.1; WOW64) ' +
'AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.92 Safari/535.2" 327 41110',
'192.168.100.200 - - [17/Nov/2015:22:09:26 +0100] "POST /wp-cron.php?doing_wp_cron=1447794566.' +
'5160338878631591796875 HTTP/1.0" 200 0 "-" "WordPress/4.3.1; http://my.domain.at.private.com" 281 0'
]
collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counter keys
counter = metrics['counter']
for key in ['C|nginx.http.method.post', 'C|nginx.http.method.get', 'C|nginx.http.status.2xx',
'C|nginx.http.v1_1', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.request.length',
'C|nginx.http.v1_0']:
assert_that(counter, has_key(key))
# values
assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(1))
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(1))
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2))
assert_that(counter['C|nginx.http.request.length'][0][1], equal_to(608))
assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1], equal_to(41110))
# check zero values
for counter_name, counter_key in collector.counters.iteritems():
if counter_key in collector.parser.keys:
assert_that(counter, has_key('C|nginx.%s' % counter_name))
if counter_name not in (
'http.status.2xx',
'http.method.post',
'http.method.get',
'http.request.length',
'http.request.body_bytes_sent',
'http.v1_1',
'http.v1_0'
):
assert_that(counter['C|nginx.%s' % counter_name][0][1], equal_to(0))
else:
if counter_key not in collector.parser.request_variables:
assert_that(counter, not_(has_key('C|nginx.%s' % counter_name)))
def test_combined(self):
lines = [
'178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /img/docker.png HTTP/1.1" 304 0 ' +
'"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' +
'"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) ' +
'Chrome/43.0.2357.124 Safari/537.36"',
'178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /api/inventory/objects/ HTTP/1.1" 200 1093 ' +
'"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' +
'"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) ' +
'Chrome/43.0.2357.124 Safari/537.36"',
'127.0.0.1 - - [18/Jun/2015:17:22:33 +0000] "POST /1.0/589fjinijenfirjf/meta/ HTTP/1.1" ' +
'202 2 "-" "python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"',
'52.6.158.18 - - [18/Jun/2015:17:22:40 +0000] "GET /#/objects HTTP/1.1" 416 84 ' +
'"-" "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)"'
]
collector = NginxAccessLogsCollector(object=self.fake_object, tail=lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counters
counter = metrics['counter']
for key in ('C|nginx.http.method.get', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.status.3xx',
'C|nginx.http.status.2xx','C|nginx.http.method.post', 'C|nginx.http.v1_1',
'C|nginx.http.status.4xx'):
assert_that(counter, has_key(key))
# values
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(3))
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2))
assert_that(counter['C|nginx.http.v1_1'][0][1], equal_to(4))
assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1], equal_to(84 + 2 + 1093 + 0))
# check zero values
for counter_name, counter_key in collector.counters.iteritems():
if counter_key in collector.parser.keys:
assert_that(counter, has_key('C|nginx.%s' % counter_name))
if counter_name not in (
'http.status.2xx',
'http.status.3xx',
'http.status.4xx',
'http.v1_1',
'http.request.body_bytes_sent'
):
assert_that(counter['C|nginx.%s' % counter_name][0][1], equal_to(0))
else:
if counter_key not in collector.parser.request_variables:
assert_that(counter, not_(has_key('C|nginx.%s' % counter_name)))
def test_http_status(self):
line = '127.0.0.1 - - [02/Jul/2015:14:49:48 +0000] "GET /basic_status HTTP/1.1" 200 110 "-" ' + \
'"python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"'
# run single method
collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
collector.http_status(NginxAccessLogParser().parse(line))
# check
metrics = self.fake_object.statsd.current
assert_that(metrics, has_item('counter'))
counters = metrics['counter']
assert_that(counters, has_item('nginx.http.status.2xx'))
assert_that(counters['nginx.http.status.2xx'][0][1], equal_to(1))
def test_extend_duplicates(self):
"""
Test a log format that defines duplicate variables (NAAS-686).
"""
log_format = '$remote_addr - $remote_addr - $remote_addr - $remote_user [$time_local] ' + \
'"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status'
lines = [
'1.2.3.4 - 1.2.3.4 - 1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' +
'"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS',
'1.2.3.4 - 1.2.3.4 - 1.2.3.4 - - [22/Jan/2010:20:34:21 +0300] "GET /foo/ HTTP/1.1" 300 1078 ' +
'"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.002" cs=HIT',
]
collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
assert_that(metrics, has_item('timer'))
# counter keys
counter = metrics['counter']
for key in ['C|nginx.http.method.get', 'C|nginx.http.v1_1', 'C|nginx.upstream.next.count',
'C|nginx.upstream.request.count', 'C|nginx.http.status.3xx', 'C|nginx.cache.miss',
'C|nginx.http.status.2xx', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.cache.hit']:
assert_that(counter, has_key(key))
# timer keys
timer = metrics['timer']
for key in ['G|nginx.upstream.response.time.pctl95', 'C|nginx.upstream.response.time.count',
'C|nginx.http.request.time.count', 'G|nginx.http.request.time',
'G|nginx.http.request.time.pctl95', 'G|nginx.http.request.time.median',
'G|nginx.http.request.time.max', 'G|nginx.upstream.response.time',
'G|nginx.upstream.response.time.median', 'G|nginx.upstream.response.time.max']:
assert_that(timer, has_key(key))
# values
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(2))
assert_that(counter['C|nginx.upstream.request.count'][0][1], equal_to(2))
assert_that(counter['C|nginx.upstream.next.count'][0][1], equal_to(1))
assert_that(timer['G|nginx.upstream.response.time.max'][0][1], equal_to(2.001+0.345))
def test_empty_upstreams(self):
log_format = '$remote_addr - $remote_user [$time_local] ' + \
'"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
'rt=$request_time cs=$upstream_cache_status ut="$upstream_response_time"'
line = \
'1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \
'"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 cs=- ut="-"'
# run single method
collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
collector.upstreams(NginxAccessLogParser(log_format).parse(line))
# check
metrics = self.fake_object.statsd.current
assert_that(metrics, equal_to(defaultdict()))
# counters
counters = metrics['counter']
assert_that(counters, equal_to({}))
# histogram
histogram = metrics['timer']
assert_that(histogram, equal_to({}))
