def test_server_name(self):
self.fake_object.filters = [
Filter(**dict(filter_rule_id=2,
metric='nginx.http.status.2xx',
data=[{
u'$server_name': u'differentsimgirls.com'
}]))
]
collector = NginxAccessLogsCollector(
object=self.fake_object,
log_format=
'$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent '
+
'\"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\" '
+
'rt=$request_time ua=\"$upstream_addr\" us=\"$upstream_status\" ' +
'ut=\"$upstream_response_time\" ul=\"$upstream_response_length\" '
+ 'cs=$upstream_cache_status sn=$server_name',
tail=[
'104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 200 28275 "-" '
+
'"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="-" ul="-" cs=- sn=differentsimgirls.com'
])
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
counter = metrics['counter']
# check our metric
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(1))
assert_that(counter['C|nginx.http.status.2xx||2'][0][1], equal_to(1))
def test_server_name(self):
self.fake_object.filters = [
Filter(**dict(
filter_rule_id=2,
metric='nginx.http.status.2xx',
data=[
{u'$server_name': u'differentsimgirls.com'}
]
))
]
collector = NginxAccessLogsCollector(
object=self.fake_object,
log_format='$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent ' +
'\"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\" ' +
'rt=$request_time ua=\"$upstream_addr\" us=\"$upstream_status\" ' +
'ut=\"$upstream_response_time\" ul=\"$upstream_response_length\" ' +
'cs=$upstream_cache_status sn=$server_name',
tail=[
'104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 200 28275 "-" ' +
'"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="-" ul="-" cs=- sn=differentsimgirls.com'
]
)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
counter = metrics['counter']
# check our metric
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(1))
assert_that(counter['C|nginx.http.status.2xx||2'][0][1], equal_to(1))
def test_upstreams(self):
log_format = '$remote_addr - $remote_user [$time_local] ' + \
'"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status'
line = \
'1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \
'"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS'
# run single method
collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
collector.upstreams(NginxAccessLogParser(log_format).parse(line))
# check
metrics = self.fake_object.statsd.current
assert_that(metrics, has_item('counter'))
assert_that(metrics, has_item('timer'))
# counters
counters = metrics['counter']
assert_that(counters, has_item('nginx.upstream.request.count'))
assert_that(counters, has_item('nginx.upstream.next.count'))
assert_that(counters, has_item('nginx.cache.miss'))
assert_that(counters['nginx.upstream.request.count'][0][1],
equal_to(1))
assert_that(counters['nginx.upstream.next.count'][0][1], equal_to(1))
assert_that(counters['nginx.cache.miss'][0][1], equal_to(1))
# histogram
histogram = metrics['timer']
assert_that(histogram, has_item('nginx.upstream.response.time'))
assert_that(histogram['nginx.upstream.response.time'],
equal_to([2.001 + 0.345]))
def test_upstreams(self):
log_format = '$remote_addr - $remote_user [$time_local] ' + \
'"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status'
line = \
'1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \
'"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS'
# run single method
collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
collector.upstreams(NginxAccessLogParser(log_format).parse(line))
# check
metrics = self.fake_object.statsd.current
assert_that(metrics, has_item('counter'))
assert_that(metrics, has_item('timer'))
# counters
counters = metrics['counter']
assert_that(counters, has_item('nginx.upstream.request.count'))
assert_that(counters, has_item('nginx.upstream.next.count'))
assert_that(counters, has_item('nginx.cache.miss'))
assert_that(counters['nginx.upstream.request.count'][0][1], equal_to(1))
assert_that(counters['nginx.upstream.next.count'][0][1], equal_to(1))
assert_that(counters['nginx.cache.miss'][0][1], equal_to(1))
# histogram
histogram = metrics['timer']
assert_that(histogram, has_item('nginx.upstream.response.time'))
assert_that(histogram['nginx.upstream.response.time'], equal_to([2.001 + 0.345]))
def test_regex_filter(self):
self.fake_object.filters = [
Filter(**dict(
filter_rule_id=2,
metric='nginx.http.request.body_bytes_sent',
data=[
{'$request_uri': '/img.*'},
{'$server_protocol': 'HTTP/1.2'}
]
))
]
collector = NginxAccessLogsCollector(object=self.fake_object, tail=self.lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counters
counter = metrics['counter']
for key in ('C|nginx.http.method.get', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.status.3xx',
'C|nginx.http.status.2xx', 'C|nginx.http.method.post', 'C|nginx.http.v1_1',
'C|nginx.http.status.4xx', 'C|nginx.http.request.body_bytes_sent||2', 'C|nginx.http.method.post'):
assert_that(counter, has_key(key))
# values
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(4))
assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(2))
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(3))
# filter values
assert_that(counter['C|nginx.http.request.body_bytes_sent||2'][0][1], equal_to(2))
def test_combined(self):
lines = [
'178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /img/docker.png HTTP/1.1" 304 0 '
+
'"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' +
'"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) '
+ 'Chrome/43.0.2357.124 Safari/537.36"',
'178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /api/inventory/objects/ HTTP/1.1" 200 1093 '
+
'"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' +
'"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) '
+ 'Chrome/43.0.2357.124 Safari/537.36"',
'127.0.0.1 - - [18/Jun/2015:17:22:33 +0000] "POST /1.0/589fjinijenfirjf/meta/ HTTP/1.1" '
+
'202 2 "-" "python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"',
'52.6.158.18 - - [18/Jun/2015:17:22:40 +0000] "GET /#/objects HTTP/1.1" 416 84 '
+
'"-" "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)"'
]
collector = NginxAccessLogsCollector(object=self.fake_object,
tail=lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counters
counter = metrics['counter']
for key in ('C|nginx.http.method.get',
'C|nginx.http.request.body_bytes_sent',
'C|nginx.http.status.3xx', 'C|nginx.http.status.2xx',
'C|nginx.http.method.post', 'C|nginx.http.v1_1',
'C|nginx.http.status.4xx'):
assert_that(counter, has_key(key))
# values
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(3))
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2))
assert_that(counter['C|nginx.http.v1_1'][0][1], equal_to(4))
assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1],
equal_to(84 + 2 + 1093 + 0))
# check zero values
for counter_name, counter_key in collector.counters.iteritems():
if counter_key in collector.parser.keys:
assert_that(counter, has_key('C|%s' % counter_name))
if counter_name not in ('nginx.http.status.2xx',
'nginx.http.status.3xx',
'nginx.http.status.4xx',
'nginx.http.v1_1',
'nginx.http.request.body_bytes_sent'):
assert_that(counter['C|%s' % counter_name][0][1],
equal_to(0))
else:
if counter_key not in collector.parser.request_variables:
assert_that(counter, not_(has_key('C|%s' % counter_name)))
def test_combined(self):
lines = [
'178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /img/docker.png HTTP/1.1" 304 0 ' +
'"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' +
'"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) ' +
'Chrome/43.0.2357.124 Safari/537.36"',
'178.23.225.78 - - [18/Jun/2015:17:22:25 +0000] "GET /api/inventory/objects/ HTTP/1.1" 200 1093 ' +
'"http://ec2-54-78-3-178.eu-west-1.compute.amazonaws.com:4000/" ' +
'"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) ' +
'Chrome/43.0.2357.124 Safari/537.36"',
'127.0.0.1 - - [18/Jun/2015:17:22:33 +0000] "POST /1.0/589fjinijenfirjf/meta/ HTTP/1.1" ' +
'202 2 "-" "python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"',
'52.6.158.18 - - [18/Jun/2015:17:22:40 +0000] "GET /#/objects HTTP/1.1" 416 84 ' +
'"-" "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)"'
]
collector = NginxAccessLogsCollector(object=self.fake_object, tail=lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counters
counter = metrics['counter']
for key in ('C|nginx.http.method.get', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.status.3xx',
'C|nginx.http.status.2xx','C|nginx.http.method.post', 'C|nginx.http.v1_1',
'C|nginx.http.status.4xx'):
assert_that(counter, has_key(key))
# values
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(3))
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2))
assert_that(counter['C|nginx.http.v1_1'][0][1], equal_to(4))
assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1], equal_to(84 + 2 + 1093 + 0))
# check zero values
for counter_name, counter_key in collector.counters.iteritems():
if counter_key in collector.parser.keys:
assert_that(counter, has_key('C|%s' % counter_name))
if counter_name not in (
'nginx.http.status.2xx',
'nginx.http.status.3xx',
'nginx.http.status.4xx',
'nginx.http.v1_1',
'nginx.http.request.body_bytes_sent'
):
assert_that(counter['C|%s' % counter_name][0][1], equal_to(0))
else:
if counter_key not in collector.parser.request_variables:
assert_that(counter, not_(has_key('C|%s' % counter_name)))
def test_http_status(self):
line = '127.0.0.1 - - [02/Jul/2015:14:49:48 +0000] "GET /basic_status HTTP/1.1" 200 110 "-" ' + \
'"python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"'
# run single method
collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
collector.http_status(NginxAccessLogParser().parse(line))
# check
metrics = self.fake_object.statsd.current
assert_that(metrics, has_item('counter'))
counters = metrics['counter']
assert_that(counters, has_item('nginx.http.status.2xx'))
assert_that(counters['nginx.http.status.2xx'][0][1], equal_to(1))
def test_http_status(self):
line = '127.0.0.1 - - [02/Jul/2015:14:49:48 +0000] "GET /basic_status HTTP/1.1" 200 110 "-" ' + \
'"python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic"'
# run single method
collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
collector.http_status(NginxAccessLogParser().parse(line))
# check
metrics = self.fake_object.statsd.current
assert_that(metrics, has_item('counter'))
counters = metrics['counter']
assert_that(counters, has_item('nginx.http.status.2xx'))
assert_that(counters['nginx.http.status.2xx'][0][1], equal_to(1))
def test_simple_filter(self):
self.fake_object.filters = [
Filter(**dict(filter_rule_id=1,
metric='nginx.http.status.2xx',
data=[{
'$request_method': 'GET'
}, {
'$status': '200'
}])),
Filter(**dict(filter_rule_id=2,
metric='nginx.http.request.body_bytes_sent',
data=[{
'$request_uri': '/img*'
}, {
'$server_protocol': 'HTTP/1.2'
}]))
]
collector = NginxAccessLogsCollector(object=self.fake_object,
tail=self.lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counters
counter = metrics['counter']
for key in ('C|nginx.http.method.get',
'C|nginx.http.request.body_bytes_sent',
'C|nginx.http.status.3xx', 'C|nginx.http.status.2xx',
'C|nginx.http.method.post', 'C|nginx.http.v1_1',
'C|nginx.http.status.4xx', 'C|nginx.http.status.2xx||1',
'C|nginx.http.method.post'):
assert_that(counter, has_key(key))
# values
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(4))
assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(2))
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(3))
# filter values
assert_that(counter['C|nginx.http.status.2xx||1'][0][1], equal_to(2))
def test_empty_upstreams(self):
log_format = '$remote_addr - $remote_user [$time_local] ' + \
'"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
'rt=$request_time cs=$upstream_cache_status ut="$upstream_response_time"'
line = \
'1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \
'"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 cs=- ut="-"'
# run single method
collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
collector.upstreams(NginxAccessLogParser(log_format).parse(line))
# check
metrics = self.fake_object.statsd.current
assert_that(metrics, equal_to(defaultdict()))
# counters
counters = metrics['counter']
assert_that(counters, equal_to({}))
# histogram
histogram = metrics['timer']
assert_that(histogram, equal_to({}))
def test_empty_upstreams(self):
log_format = '$remote_addr - $remote_user [$time_local] ' + \
'"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
'rt=$request_time cs=$upstream_cache_status ut="$upstream_response_time"'
line = \
'1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' + \
'"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 cs=- ut="-"'
# run single method
collector = NginxAccessLogsCollector(object=self.fake_object, tail=[])
collector.upstreams(NginxAccessLogParser(log_format).parse(line))
# check
metrics = self.fake_object.statsd.current
assert_that(metrics, equal_to(defaultdict()))
# counters
counters = metrics['counter']
assert_that(counters, equal_to({}))
# histogram
histogram = metrics['timer']
assert_that(histogram, equal_to({}))
def __init__(self, **kwargs):
super(NginxObject, self).__init__(**kwargs)
self.root_uuid = self.data.get('root_uuid') or context.objects.root_object.uuid if context.objects.root_object else None
self.local_id_cache = self.data['local_id'] # Assigned by manager
self.pid = self.data['pid']
self.version = self.data['version']
self.workers = self.data['workers']
self.prefix = self.data['prefix']
self.bin_path = self.data['bin_path']
self.conf_path = self.data['conf_path']
default_config = context.app_config['containers'][self.type]
self.upload_config = self.data.get('upload_config') or default_config.get('upload_config', False)
self.run_config_test = self.data.get('run_test') or default_config.get('run_test', False)
self.upload_ssl = self.data.get('upload_ssl') or default_config.get('upload_ssl', False)
self.config = NginxConfig(self.conf_path, prefix=self.prefix)
self.config.full_parse()
self.plus_status_external_url, self.plus_status_internal_url = self.get_alive_plus_status_urls()
self.plus_status_enabled = True if (self.plus_status_external_url or self.plus_status_internal_url) else False
self.stub_status_url = self.get_alive_stub_status_url()
self.stub_status_enabled = True if self.stub_status_url else False
self.processes = []
self.filters = []
# filters
for raw_filter in self.data.get('filters') or []:
self.filters.append(Filter(**raw_filter))
# meta
meta_collector_class = NginxCommonMetaCollector
if host.os_name() == 'linux':
if host.linux_name() in ('ubuntu', 'debian'):
meta_collector_class = NginxDebianMetaCollector
elif host.linux_name() in ('centos',):
meta_collector_class = NginxCentosMetaCollector
self.collectors = [
meta_collector_class(
object=self, interval=self.intervals['meta']
),
NginxMetricsCollector(
object=self, interval=self.intervals['metrics']
),
NginxConfigCollector(
object=self, interval=self.intervals['configs'],
)
]
# access logs
for log_filename, format_name in self.config.access_logs.iteritems():
log_format = self.config.log_formats.get(format_name)
try:
self.collectors.append(
NginxAccessLogsCollector(
object=self,
interval=self.intervals['logs'],
filename=log_filename,
log_format=log_format,
)
)
# Send access log discovery event.
self.eventd.event(level=INFO, message='nginx access log %s found' % log_filename)
except IOError as e:
exception_name = e.__class__.__name__
context.log.error(
'failed to start reading log %s due to %s (maybe has no rights?)' %
(log_filename, exception_name)
)
context.log.debug('additional info:', exc_info=True)
# error logs
for log_filename, log_level in self.config.error_logs.iteritems():
try:
self.collectors.append(
NginxErrorLogsCollector(
object=self,
interval=self.intervals['logs'],
filename=log_filename,
level=log_level
)
)
# Send error log discovery event.
self.eventd.event(level=INFO, message='nginx error log %s found' % log_filename)
except IOError as e:
exception_name = e.__class__.__name__
context.log.error(
'failed to start reading log %s due to %s (maybe has no rights?)' %
(log_filename, exception_name)
)
context.log.debug('additional info:', exc_info=True)
def test_extended(self):
log_format = '$remote_addr - $remote_user [$time_local] ' + \
'"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status'
lines = [
'1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 '
+
'"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS',
'1.2.3.4 - - [22/Jan/2010:20:34:21 +0300] "GET /foo/ HTTP/1.1" 300 1078 '
+
'"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.002" cs=HIT',
]
collector = NginxAccessLogsCollector(object=self.fake_object,
log_format=log_format,
tail=lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
assert_that(metrics, has_item('timer'))
# counter keys
counter = metrics['counter']
for key in [
'C|nginx.http.method.get', 'C|nginx.http.v1_1',
'C|nginx.upstream.next.count',
'C|nginx.upstream.request.count', 'C|nginx.http.status.3xx',
'C|nginx.cache.miss', 'C|nginx.http.status.2xx',
'C|nginx.http.request.body_bytes_sent', 'C|nginx.cache.hit'
]:
assert_that(counter, has_key(key))
# timer keys
timer = metrics['timer']
for key in [
'G|nginx.upstream.response.time.pctl95',
'C|nginx.upstream.response.time.count',
'C|nginx.http.request.time.count', 'G|nginx.http.request.time',
'G|nginx.http.request.time.pctl95',
'G|nginx.http.request.time.median',
'G|nginx.http.request.time.max',
'G|nginx.upstream.response.time',
'G|nginx.upstream.response.time.median',
'G|nginx.upstream.response.time.max'
]:
assert_that(timer, has_key(key))
# values
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(2))
assert_that(counter['C|nginx.upstream.request.count'][0][1],
equal_to(2))
assert_that(counter['C|nginx.upstream.next.count'][0][1], equal_to(1))
assert_that(timer['G|nginx.upstream.response.time.max'][0][1],
equal_to(2.001 + 0.345))
# check zero values
for counter_name, counter_key in collector.counters.iteritems():
if counter_key in collector.parser.keys:
assert_that(counter, has_key('C|%s' % counter_name))
if counter_name not in ('nginx.http.status.2xx',
'nginx.http.status.3xx',
'nginx.http.method.get',
'nginx.upstream.request.count',
'nginx.upstream.next.count',
'nginx.upstream.response.time.max',
'nginx.http.v1_1', 'nginx.cache.miss',
'nginx.cache.hit',
'nginx.http.request.body_bytes_sent'):
assert_that(counter['C|%s' % counter_name][0][1],
equal_to(0))
else:
if counter_key not in collector.parser.request_variables:
assert_that(counter, not_(has_key('C|%s' % counter_name)))
def test_extend_duplicates_reported(self):
"""
Test the specific reported bug format.
"""
log_format = '$remote_addr - $remote_user [$time_local] ' + \
'"$request" $status $body_bytes_sent ' + \
'"$http_referer" "$http_user_agent" ' + \
'$request_length $body_bytes_sent'
lines = [
'188.165.1.1 - - [17/Nov/2015:22:07:42 +0100] ' +
'"GET /2014/09/quicktipp-phpmyadmin-update-script/?pk_campaign=feed&pk_kwd=quicktipp-'
+ 'phpmyadmin-update-script HTTP/1.1" ' +
'200 41110 "http://www.google.co.uk/url?sa=t&source=web&cd=1" "Mozilla/5.0 (Windows NT 6.1; WOW64) '
+
'AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.92 Safari/535.2" 327 41110',
'192.168.100.200 - - [17/Nov/2015:22:09:26 +0100] "POST /wp-cron.php?doing_wp_cron=1447794566.'
+
'5160338878631591796875 HTTP/1.0" 200 0 "-" "WordPress/4.3.1; http://my.domain.at.private.com" 281 0'
]
collector = NginxAccessLogsCollector(object=self.fake_object,
log_format=log_format,
tail=lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counter keys
counter = metrics['counter']
for key in [
'C|nginx.http.method.post', 'C|nginx.http.method.get',
'C|nginx.http.status.2xx', 'C|nginx.http.v1_1',
'C|nginx.http.request.body_bytes_sent', 'C|nginx.http.v1_0'
]:
assert_that(counter, has_key(key))
# average keys
averages = metrics['average']
assert_that(averages, has_key('G|nginx.http.request.length'))
# values
assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(1))
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(1))
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2))
assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1],
equal_to(41110))
assert_that(averages['G|nginx.http.request.length'][0][1],
equal_to((327 + 281) / 2))
# check zero values
for counter_name, counter_key in collector.counters.iteritems():
if counter_key in collector.parser.keys:
assert_that(counter, has_key('C|%s' % counter_name))
if counter_name not in ('nginx.http.status.2xx',
'nginx.http.method.post',
'nginx.http.method.get',
'nginx.http.request.length',
'nginx.http.request.body_bytes_sent',
'nginx.http.v1_1', 'nginx.http.v1_0'):
assert_that(counter['C|%s' % counter_name][0][1],
equal_to(0))
else:
if counter_key not in collector.parser.request_variables:
assert_that(counter, not_(has_key('C|%s' % counter_name)))
def test_extended(self):
log_format = '$remote_addr - $remote_user [$time_local] ' + \
'"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + \
'rt=$request_time ut="$upstream_response_time" cs=$upstream_cache_status'
lines = [
'1.2.3.4 - - [22/Jan/2010:19:34:21 +0300] "GET /foo/ HTTP/1.1" 200 11078 ' +
'"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.001, 0.345" cs=MISS',
'1.2.3.4 - - [22/Jan/2010:20:34:21 +0300] "GET /foo/ HTTP/1.1" 300 1078 ' +
'"http://www.rambler.ru/" "Mozilla/5.0 (Windows; U; Windows NT 5.1" rt=0.010 ut="2.002" cs=HIT',
]
collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
assert_that(metrics, has_item('timer'))
# counter keys
counter = metrics['counter']
for key in ['C|nginx.http.method.get', 'C|nginx.http.v1_1', 'C|nginx.upstream.next.count',
'C|nginx.upstream.request.count', 'C|nginx.http.status.3xx', 'C|nginx.cache.miss',
'C|nginx.http.status.2xx', 'C|nginx.http.request.body_bytes_sent', 'C|nginx.cache.hit']:
assert_that(counter, has_key(key))
# timer keys
timer = metrics['timer']
for key in ['G|nginx.upstream.response.time.pctl95', 'C|nginx.upstream.response.time.count',
'C|nginx.http.request.time.count', 'G|nginx.http.request.time',
'G|nginx.http.request.time.pctl95', 'G|nginx.http.request.time.median',
'G|nginx.http.request.time.max', 'G|nginx.upstream.response.time',
'G|nginx.upstream.response.time.median', 'G|nginx.upstream.response.time.max']:
assert_that(timer, has_key(key))
# values
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(2))
assert_that(counter['C|nginx.upstream.request.count'][0][1], equal_to(2))
assert_that(counter['C|nginx.upstream.next.count'][0][1], equal_to(1))
assert_that(timer['G|nginx.upstream.response.time.max'][0][1], equal_to(2.001+0.345))
# check zero values
for counter_name, counter_key in collector.counters.iteritems():
if counter_key in collector.parser.keys:
assert_that(counter, has_key('C|%s' % counter_name))
if counter_name not in (
'nginx.http.status.2xx',
'nginx.http.status.3xx',
'nginx.http.method.get',
'nginx.upstream.request.count',
'nginx.upstream.next.count',
'nginx.upstream.response.time.max',
'nginx.http.v1_1',
'nginx.cache.miss',
'nginx.cache.hit',
'nginx.http.request.body_bytes_sent'
):
assert_that(counter['C|%s' % counter_name][0][1], equal_to(0))
else:
if counter_key not in collector.parser.request_variables:
assert_that(counter, not_(has_key('C|%s' % counter_name)))
def test_extend_duplicates_reported(self):
"""
Test the specific reported bug format.
"""
log_format = '$remote_addr - $remote_user [$time_local] ' + \
'"$request" $status $body_bytes_sent ' + \
'"$http_referer" "$http_user_agent" ' + \
'$request_length $body_bytes_sent'
lines = [
'188.165.1.1 - - [17/Nov/2015:22:07:42 +0100] ' +
'"GET /2014/09/quicktipp-phpmyadmin-update-script/?pk_campaign=feed&pk_kwd=quicktipp-' +
'phpmyadmin-update-script HTTP/1.1" ' +
'200 41110 "http://www.google.co.uk/url?sa=t&source=web&cd=1" "Mozilla/5.0 (Windows NT 6.1; WOW64) ' +
'AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.92 Safari/535.2" 327 41110',
'192.168.100.200 - - [17/Nov/2015:22:09:26 +0100] "POST /wp-cron.php?doing_wp_cron=1447794566.' +
'5160338878631591796875 HTTP/1.0" 200 0 "-" "WordPress/4.3.1; http://my.domain.at.private.com" 281 0'
]
collector = NginxAccessLogsCollector(object=self.fake_object, log_format=log_format, tail=lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counter keys
counter = metrics['counter']
for key in ['C|nginx.http.method.post', 'C|nginx.http.method.get', 'C|nginx.http.status.2xx',
'C|nginx.http.v1_1', 'C|nginx.http.request.body_bytes_sent',
'C|nginx.http.v1_0']:
assert_that(counter, has_key(key))
# average keys
averages = metrics['average']
assert_that(averages, has_key('G|nginx.http.request.length'))
# values
assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(1))
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(1))
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(2))
assert_that(counter['C|nginx.http.request.body_bytes_sent'][0][1], equal_to(41110))
assert_that(averages['G|nginx.http.request.length'][0][1], equal_to((327+281)/2))
# check zero values
for counter_name, counter_key in collector.counters.iteritems():
if counter_key in collector.parser.keys:
assert_that(counter, has_key('C|%s' % counter_name))
if counter_name not in (
'nginx.http.status.2xx',
'nginx.http.method.post',
'nginx.http.method.get',
'nginx.http.request.length',
'nginx.http.request.body_bytes_sent',
'nginx.http.v1_1',
'nginx.http.v1_0'
):
assert_that(counter['C|%s' % counter_name][0][1], equal_to(0))
else:
if counter_key not in collector.parser.request_variables:
assert_that(counter, not_(has_key('C|%s' % counter_name)))