def test_filter_with_negated(self):
self.fake_object.filters = [
Filter(filter_rule_id=1, metric='nginx.http.method.get', data=[['$status', '~', '200']]),
Filter(filter_rule_id=2, metric='nginx.http.method.get', data=[['$status', '!~', '200']]),
Filter(filter_rule_id=3, metric='nginx.http.method.get', data=[['$status', '~', '304']]),
Filter(filter_rule_id=4, metric='nginx.http.method.get', data=[['$status', '!~', '304']]),
]
collector = NginxAccessLogsCollector(object=self.fake_object, tail=self.lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counters
counter = metrics['counter']
assert_that(counter, has_key('C|nginx.http.method.get'))
assert_that(counter['C|nginx.http.method.get'], contains(contains(is_(int), 4)))
# filter values
assert_that(counter, has_key('C|nginx.http.method.get||1'))
assert_that(counter, has_key('C|nginx.http.method.get||2'))
assert_that(counter, has_key('C|nginx.http.method.get||3'))
assert_that(counter, has_key('C|nginx.http.method.get||4'))
assert_that(counter['C|nginx.http.method.get||1'], contains(contains(is_(int), 2)))
assert_that(counter['C|nginx.http.method.get||2'], contains(contains(is_(int), 2)))
assert_that(counter['C|nginx.http.method.get||3'], contains(contains(is_(int), 1)))
assert_that(counter['C|nginx.http.method.get||4'], contains(contains(is_(int), 3)))
def test_unused_filter_defaults_zero(self):
self.fake_object.filters = [
Filter(filter_rule_id=1, metric='nginx.http.status.4xx', data=[['status', '~', '416']]),
Filter(filter_rule_id=2, metric='nginx.http.status.4xx', data=[['status', '~', '300']]),
Filter(filter_rule_id=3, metric='nginx.http.method.get', data=[['$request_uri', '~', '/img.*']]),
Filter(filter_rule_id=4, metric='nginx.http.method.get', data=[['$request_uri', '~', 'abcdef']]),
]
collector = NginxAccessLogsCollector(object=self.fake_object, tail=self.lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counters
counter = metrics['counter']
# filter values
assert_that(counter, has_key('C|nginx.http.status.4xx||1'))
assert_that(counter, has_key('C|nginx.http.status.4xx||2'))
assert_that(counter, has_key('C|nginx.http.method.get||3'))
assert_that(counter, has_key('C|nginx.http.method.get||4'))
assert_that(counter['C|nginx.http.status.4xx||1'], contains(contains(is_(int), 1)))
assert_that(counter['C|nginx.http.status.4xx||2'], contains(contains(is_(int), 0)))
assert_that(counter['C|nginx.http.method.get||3'], contains(contains(is_(int), 2)))
assert_that(counter['C|nginx.http.method.get||4'], contains(contains(is_(int), 0)))
def test_regex_filter(self):
self.fake_object.filters = [
Filter(filter_rule_id=2,
metric='nginx.http.request.body_bytes_sent',
data=[['$request_uri', '~', '/img.*'],
['$server_protocol', '~', 'HTTP/1.2']])
]
collector = NginxAccessLogsCollector(object=self.fake_object,
tail=self.lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counters
counter = metrics['counter']
for key in ('C|nginx.http.method.get',
'C|nginx.http.request.body_bytes_sent',
'C|nginx.http.status.3xx', 'C|nginx.http.status.2xx',
'C|nginx.http.method.post', 'C|nginx.http.v1_1',
'C|nginx.http.status.4xx',
'C|nginx.http.request.body_bytes_sent||2',
'C|nginx.http.method.post'):
assert_that(counter, has_key(key))
# values
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(5))
assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(2))
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(3))
# filter values
assert_that(counter['C|nginx.http.request.body_bytes_sent||2'][0][1],
equal_to(2))
def test_server_name(self):
self.fake_object.filters = [
Filter(filter_rule_id=2,
metric='nginx.http.status.2xx',
data=[['$server_name', '~', 'differentsimgirls.com']])
]
collector = NginxAccessLogsCollector(
object=self.fake_object,
log_format=
'$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent '
+
'\"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\" '
+
'rt=$request_time ua=\"$upstream_addr\" us=\"$upstream_status\" ' +
'ut=\"$upstream_response_time\" ul=\"$upstream_response_length\" '
+ 'cs=$upstream_cache_status sn=$server_name',
tail=[
'104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 200 28275 "-" '
+
'"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="-" ul="-" cs=- sn=differentsimgirls.com'
])
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
counter = metrics['counter']
# check our metric
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(1))
assert_that(counter['C|nginx.http.status.2xx||2'][0][1], equal_to(1))
def test_separate_4xx_5xx_with_filters(self):
self.fake_object.filters = [
Filter(filter_rule_id=1,
metric='nginx.http.status.502',
data=[
['$request_method', '~', 'GET'],
])
]
collector = NginxAccessLogsCollector(object=self.fake_object,
tail=self.lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counters
counter = metrics['counter']
for key in ('C|nginx.http.method.get',
'C|nginx.http.request.body_bytes_sent',
'C|nginx.http.status.3xx', 'C|nginx.http.status.2xx',
'C|nginx.http.method.post', 'C|nginx.http.v1_1',
'C|nginx.http.status.4xx', 'C|nginx.http.status.5xx',
'C|nginx.http.status.502||1', 'C|nginx.http.method.post'):
assert_that(counter, has_key(key))
# values
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(5))
assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(2))
assert_that(counter['C|nginx.http.status.5xx'][0][1], equal_to(1))
# filter values
assert_that(counter['C|nginx.http.status.502||1'][0][1], equal_to(1))
def __init__(self, **kwargs):
super(NginxObject, self).__init__(**kwargs)
# Have to override intervals here because new container sub objects.
self.intervals = context.app_config['containers'].get('nginx', {}).get(
'poll_intervals', {'default': 10})
self.root_uuid = self.data.get('root_uuid') or getattr(
context.objects.root_object, 'uuid', None)
self._local_id = self.data['local_id'] # Assigned by manager
self.pid = self.data['pid']
self.version = self.data['version']
self.workers = self.data['workers']
self.prefix = self.data['prefix']
self.bin_path = self.data['bin_path']
self.conf_path = self.data['conf_path']
self.name = self.version
# agent config
default_config = context.app_config['containers']['nginx']
self.upload_config = self.data.get(
'upload_config') or default_config.get('upload_config', False)
self.run_config_test = self.data.get('run_test') or default_config.get(
'run_test', False)
self.upload_ssl = self.data.get('upload_ssl') or default_config.get(
'upload_ssl', False)
# nginx -V data
self.parsed_v = nginx_v(self.bin_path)
# filters
self.filters = [
Filter(**raw_filter)
for raw_filter in self.data.get('filters') or []
]
# nginx config
if 'config_data' in self.data:
self._restore_config_collector(
self.data['config_data']['previous'])
else:
self._setup_config_collector()
# plus status
self.plus_status_external_url, self.plus_status_internal_url = self.get_alive_plus_status_urls(
)
self.plus_status_enabled = True if (
self.plus_status_external_url
or self.plus_status_internal_url) else False
# stub status
self.stub_status_url = self.get_alive_stub_status_url()
self.stub_status_enabled = True if self.stub_status_url else False
self.processes = []
self._setup_meta_collector()
self._setup_metrics_collector()
self._setup_access_logs()
self._setup_error_logs()
def test_empty(self):
raw_filter_data = {
'filter_rule_id': '1',
'metric': 'http.something',
'data': []
}
filtr = Filter(**raw_filter_data)
assert_that(filtr.empty, equal_to(True))
def test_init_without_filename(self):
filtr = Filter(filter_rule_id='1',
metric='http.something',
data=[{
'$request_method': 'post'
}, {
'$request_uri': '*.gif'
}])
assert_that(filtr.filename, equal_to(None))
def test_simple_filter(self):
self.fake_object.filters = [
Filter(**dict(filter_rule_id=1,
metric='nginx.http.status.2xx',
data=[{
'$request_method': 'GET'
}, {
'$status': '200'
}])),
Filter(**dict(filter_rule_id=2,
metric='nginx.http.request.body_bytes_sent',
data=[{
'$request_uri': '/img*'
}, {
'$server_protocol': 'HTTP/1.2'
}]))
]
collector = NginxAccessLogsCollector(object=self.fake_object,
tail=self.lines)
collector.collect()
# check
metrics = self.fake_object.statsd.flush()['metrics']
assert_that(metrics, has_item('counter'))
# counters
counter = metrics['counter']
for key in ('C|nginx.http.method.get',
'C|nginx.http.request.body_bytes_sent',
'C|nginx.http.status.3xx', 'C|nginx.http.status.2xx',
'C|nginx.http.method.post', 'C|nginx.http.v1_1',
'C|nginx.http.status.4xx', 'C|nginx.http.status.2xx||1',
'C|nginx.http.method.post'):
assert_that(counter, has_key(key))
# values
assert_that(counter['C|nginx.http.method.get'][0][1], equal_to(4))
assert_that(counter['C|nginx.http.method.post'][0][1], equal_to(2))
assert_that(counter['C|nginx.http.status.2xx'][0][1], equal_to(3))
# filter values
assert_that(counter['C|nginx.http.status.2xx||1'][0][1], equal_to(2))
def test_init_without_filename(self):
raw_filter_data = {
'filter_rule_id': '1',
'metric': 'http.something',
'data': [{
'$request_method': 'post'
}, {
'$request_uri': '*.gif'
}]
}
filtr = Filter(**raw_filter_data)
assert_that(filtr.filename, equal_to(None))
def test_init(self):
filtr = Filter(filter_rule_id='1',
metric='http.something',
data=[{
'logname': 'foo.txt'
}, {
'$request_method': 'post'
}, {
'$request_uri': '.*\.gif'
}])
assert_that(filtr.filter_rule_id, equal_to('1'))
assert_that(filtr.metric, equal_to('http.something'))
assert_that(filtr.filename, equal_to('foo.txt'))
assert_that(filtr.data['request_method'], equal_to(re.compile("POST")))
assert_that(filtr.data['request_uri'], equal_to(re.compile(".*\.gif")))
def test_filematch(self):
filtr = Filter(filter_rule_id='1',
metric='http.something',
data=[['logname', '~', 'foo.txt']])
assert_that(filtr.matchfile('foo.txt'), equal_to(True))
assert_that(filtr.matchfile('foo.log'), equal_to(False))
filtr = Filter(filter_rule_id='1',
metric='http.something',
data=[['logname', '!~', 'foo.txt']])
assert_that(filtr.matchfile('foo.txt'), equal_to(False))
assert_that(filtr.matchfile('foo.log'), equal_to(True))
filtr = Filter(filter_rule_id='1',
metric='http.something',
data=[['$request_method', '~', 'post'],
['$request_uri', '~', '.*\.gif'],
['$status', '!~', '200']])
assert_that(filtr.matchfile('foo.txt'), equal_to(True))
assert_that(filtr.matchfile('foo.log'), equal_to(True))
def test_init(self):
filtr = Filter(filter_rule_id='1',
metric='http.something',
data=[['logname', '~', 'foo.txt'],
['$request_method', '~', 'post'],
['$request_uri', '~', '.*\.gif'],
['$status', '!~', '200']])
assert_that(filtr.filter_rule_id, equal_to('1'))
assert_that(filtr.metric, equal_to('http.something'))
assert_that(filtr.filename, equal_to('foo.txt'))
assert_that('logname', not_(is_in(filtr.data)))
assert_that(filtr.data['request_method'], equal_to(re.compile("POST")))
assert_that(filtr.data['request_uri'], equal_to(re.compile(".*\.gif")))
assert_that(filtr.data['status'], equal_to(re.compile("200")))
assert_that('logname', not_(is_in(filtr._negated_conditions)))
assert_that(filtr._negated_conditions['request_method'],
equal_to(False))
assert_that(filtr._negated_conditions['request_uri'], equal_to(False))
assert_that(filtr._negated_conditions['status'], equal_to(True))
def test_init(self):
raw_filter_data = {
'filter_rule_id':
'1',
'metric':
'http.something',
'data': [{
'logname': 'foo.txt'
}, {
'$request_method': 'post'
}, {
'$request_uri': '.*\.gif'
}]
}
filtr = Filter(**raw_filter_data)
assert_that(filtr.filter_rule_id, equal_to('1'))
assert_that(filtr.metric, equal_to('http.something'))
assert_that(filtr.filename, equal_to('foo.txt'))
assert_that(filtr.data['request_method'], equal_to(re.compile("POST")))
assert_that(filtr.data['request_uri'], equal_to(re.compile(".*\.gif")))
def __init__(self, **kwargs):
super(NginxObject, self).__init__(**kwargs)
self.root_uuid = self.data.get('root_uuid') or context.objects.root_object.uuid if context.objects.root_object else None
self.local_id_cache = self.data['local_id'] # Assigned by manager
self.pid = self.data['pid']
self.version = self.data['version']
self.workers = self.data['workers']
self.prefix = self.data['prefix']
self.bin_path = self.data['bin_path']
self.conf_path = self.data['conf_path']
default_config = context.app_config['containers'][self.type]
self.upload_config = self.data.get('upload_config') or default_config.get('upload_config', False)
self.run_config_test = self.data.get('run_test') or default_config.get('run_test', False)
self.upload_ssl = self.data.get('upload_ssl') or default_config.get('upload_ssl', False)
self.config = NginxConfig(self.conf_path, prefix=self.prefix)
self.config.full_parse()
self.plus_status_external_url, self.plus_status_internal_url = self.get_alive_plus_status_urls()
self.plus_status_enabled = True if (self.plus_status_external_url or self.plus_status_internal_url) else False
self.stub_status_url = self.get_alive_stub_status_url()
self.stub_status_enabled = True if self.stub_status_url else False
self.processes = []
self.filters = []
# filters
for raw_filter in self.data.get('filters') or []:
self.filters.append(Filter(**raw_filter))
# meta
meta_collector_class = NginxCommonMetaCollector
if host.os_name() == 'linux':
if host.linux_name() in ('ubuntu', 'debian'):
meta_collector_class = NginxDebianMetaCollector
elif host.linux_name() in ('centos',):
meta_collector_class = NginxCentosMetaCollector
self.collectors = [
meta_collector_class(
object=self, interval=self.intervals['meta']
),
NginxMetricsCollector(
object=self, interval=self.intervals['metrics']
),
NginxConfigCollector(
object=self, interval=self.intervals['configs'],
)
]
# access logs
for log_filename, format_name in self.config.access_logs.iteritems():
log_format = self.config.log_formats.get(format_name)
try:
self.collectors.append(
NginxAccessLogsCollector(
object=self,
interval=self.intervals['logs'],
filename=log_filename,
log_format=log_format,
)
)
# Send access log discovery event.
self.eventd.event(level=INFO, message='nginx access log %s found' % log_filename)
except IOError as e:
exception_name = e.__class__.__name__
context.log.error(
'failed to start reading log %s due to %s (maybe has no rights?)' %
(log_filename, exception_name)
)
context.log.debug('additional info:', exc_info=True)
# error logs
for log_filename, log_level in self.config.error_logs.iteritems():
try:
self.collectors.append(
NginxErrorLogsCollector(
object=self,
interval=self.intervals['logs'],
filename=log_filename,
level=log_level
)
)
# Send error log discovery event.
self.eventd.event(level=INFO, message='nginx error log %s found' % log_filename)
except IOError as e:
exception_name = e.__class__.__name__
context.log.error(
'failed to start reading log %s due to %s (maybe has no rights?)' %
(log_filename, exception_name)
)
context.log.debug('additional info:', exc_info=True)
def test_timers_with_filters(self):
self.fake_object.filters = [
Filter(filter_rule_id=3,
metric='nginx.upstream.response.time.median',
data=[['$status', '~', '200']]),
Filter(filter_rule_id=4,
metric='nginx.upstream.response.time.max',
data=[['$status', '~', '400']])
]
collector = NginxAccessLogsCollector(
object=self.fake_object,
log_format=
'$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent '
+
'\"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\" '
+
'rt=$request_time ua=\"$upstream_addr\" us=\"$upstream_status\" ' +
'ut=\"$upstream_response_time\" ul=\"$upstream_response_length\" '
+ 'cs=$upstream_cache_status sn=$server_name',
tail=[
'104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 200 28275 "-" '
+
'"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="1.000" ul="-" cs=- sn=differentsimgirls.com',
'104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 200 28275 "-" '
+
'"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="3.000" ul="-" cs=- sn=differentsimgirls.com',
'104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 200 28275 "-" '
+
'"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="5.000" ul="-" cs=- sn=differentsimgirls.com',
'104.236.93.23 - - [05/May/2016:12:52:50 +0200] "GET / HTTP/1.1" 400 28275 "-" '
+
'"curl/7.35.0" "-" rt=0.082 ua="-" us="-" ut="7.000" ul="-" cs=- sn=differentsimgirls.com'
])
collector.collect()
metrics = self.fake_object.statsd.flush()['metrics']
timer = metrics['timer']
for key in [
'G|nginx.upstream.response.time.max',
'G|nginx.upstream.response.time.median',
'G|nginx.upstream.response.time.pctl95',
'C|nginx.upstream.response.time.count',
'G|nginx.upstream.response.time.max||3',
'G|nginx.upstream.response.time.median||3',
'G|nginx.upstream.response.time.pctl95||3',
'C|nginx.upstream.response.time.count||3',
'G|nginx.upstream.response.time.max||4',
'G|nginx.upstream.response.time.median||4',
'G|nginx.upstream.response.time.pctl95||4',
'C|nginx.upstream.response.time.count||4'
]:
assert_that(timer, has_key(key))
assert_that(timer["G|nginx.upstream.response.time.max||3"][0][1],
equal_to(5.000))
assert_that(timer["G|nginx.upstream.response.time.median||3"][0][1],
equal_to(3.000))
assert_that(timer["C|nginx.upstream.response.time.count||3"][0][1],
equal_to(3))
assert_that(timer["G|nginx.upstream.response.time.max||4"][0][1],
equal_to(7.000))
assert_that(timer["G|nginx.upstream.response.time.median||4"][0][1],
equal_to(7.000))
assert_that(timer["C|nginx.upstream.response.time.count||4"][0][1],
equal_to(1))
assert_that(timer["G|nginx.upstream.response.time.max"][0][1],
equal_to(7.000))
assert_that(timer["G|nginx.upstream.response.time.median"][0][1],
equal_to(4.000))
assert_that(timer["C|nginx.upstream.response.time.count"][0][1],
equal_to(4))
def test_empty(self):
filtr = Filter(filter_rule_id='1', metric='http.something', data=[])
assert_that(filtr.empty, equal_to(True))
def __init__(self, **kwargs):
super(NginxObject, self).__init__(**kwargs)
# Have to override intervals here because new container sub objects.
self.intervals = context.app_config['containers'].get('nginx', {}).get(
'poll_intervals', {'default': 10})
self.root_uuid = context.uuid
self._local_id = self.data['local_id'] # Assigned by manager
self.pid = self.data['pid']
self.version = self.data['version']
self.workers = self.data['workers']
self.prefix = self.data['prefix']
self.bin_path = self.data['bin_path']
self.conf_path = self.data['conf_path']
self.name = self.version
# agent config
default_config = context.app_config['containers']['nginx']
self.upload_config = self.data.get(
'upload_config') or default_config.get('upload_config', False)
self.run_config_test = self.data.get('run_test') or default_config.get(
'run_test', False)
self.upload_ssl = self.data.get('upload_ssl') or default_config.get(
'upload_ssl', False)
# nginx -V data
self.parsed_v = nginx_v(self.bin_path)
# filters
self.filters = [
Filter(**raw_filter)
for raw_filter in self.data.get('filters') or []
]
# nginx config
if 'config_data' in self.data:
self._restore_config_collector(
self.data['config_data']['previous'])
else:
self._setup_config_collector()
# api
self.api_endpoints_to_skip = self.get_api_endpoints_to_skip()
self.api_external_url, self.api_internal_url = self.get_alive_api_urls(
)
self.api_enabled = True if (self.api_external_url
or self.api_internal_url) else False
api_url = self.api_internal_url if self.api_internal_url is not None else self.api_external_url
if self.api_enabled and plus.get_latest_supported_api(api_url) is None:
context.log.debug(
"API directive was specified but no supported API was found.")
self.api_enabled = False
# plus status
self.plus_status_external_url, self.plus_status_internal_url = self.get_alive_plus_status_urls(
)
self.plus_status_enabled = True if (
self.plus_status_external_url
or self.plus_status_internal_url) else False
# stub status
self.stub_status_url = self.get_alive_stub_status_url()
self.stub_status_enabled = True if self.stub_status_url else False
self.processes = []
self.reloads = self.data.get('reloads', 0)
self._setup_meta_collector()
self._setup_metrics_collector()
self._setup_access_logs()
self._setup_error_logs()
# publish events for old object
for error in self.config.parser_errors:
self.eventd.event(level=WARNING, message=error)