def setUpModule():
"""
Prepare the test suite.
Sets up logging to the terminal. When a test fails the logging output can
help to perform a post-mortem analysis of the failure in question (even
when its hard to reproduce locally). This is especially useful when
debugging remote test failures, whether they happened on Travis CI or a
user's local system.
Also makes sure that the Apache web server is installed and running because
this is required to run the test suite.
"""
# Set up logging to the terminal.
coloredlogs.install(level=logging.DEBUG)
# Make sure Apache is installed and configured.
try:
manager = ApacheManager()
manager.fetch_status_page(manager.text_status_url)
except Exception as e:
logger.exception("Failed to connect to local Apache server!")
raise Exception(compact("""
Please make sure the Apache web server is installed and configured
(running) before you run this test suite because this test suite
tests the actual integration with Apache (it doesn't use mocking)
and so requires Apache to be installed, configured and running.
Swallowed exception: {message} ({type})
""", message=e, type=type(e)))
def test_save_metrics(self):
"""Test that monitoring metrics can be saved to a text file."""
fd, temporary_file = tempfile.mkstemp()
try:
manager = ApacheManager()
manager.save_metrics(temporary_file)
with open(temporary_file) as handle:
keywords = [
tokens[0] for tokens in (line.split() for line in handle)
if tokens
]
assert all(required_kw in keywords for required_kw in [
'busy-workers',
'bytes-per-request',
'bytes-per-second',
'cpu-load',
'idle-workers',
'requests-per-second',
'status-response',
'total-accesses',
'total-traffic',
'uptime',
'workers-killed-active',
'workers-killed-idle',
])
finally:
os.unlink(temporary_file)
def setUpModule():
"""
Prepare the test suite.
Sets up logging to the terminal. When a test fails the logging output can
help to perform a post-mortem analysis of the failure in question (even
when its hard to reproduce locally). This is especially useful when
debugging remote test failures, whether they happened on Travis CI or a
user's local system.
Also makes sure that the Apache web server is installed and running because
this is required to run the test suite.
"""
# Set up logging to the terminal.
coloredlogs.install(level=logging.DEBUG)
# Make sure Apache is installed and configured.
try:
manager = ApacheManager()
manager.fetch_status_page(manager.text_status_url)
except Exception as e:
logger.exception("Failed to connect to local Apache server!")
raise Exception(
compact("""
Please make sure the Apache web server is installed and configured
(running) before you run this test suite because this test suite
tests the actual integration with Apache (it doesn't use mocking)
and so requires Apache to be installed, configured and running.
Swallowed exception: {message} ({type})
""",
message=e,
type=type(e)))
def test_port_discovery(self):
"""Test Apache port discovery and error handling."""
# Test that port discovery raises an exception when ports.conf doesn't exist.
config_file = 'this-ports-config-does-not-exist-%i.conf' % os.getpid()
manager = ApacheManager(
os.path.join(tempfile.gettempdir(), config_file))
self.assertRaises(AddressDiscoveryError,
lambda: manager.listen_addresses)
# Test that port discovery raises an exception when parsing of ports.conf fails.
with tempfile.NamedTemporaryFile() as temporary_file:
manager = ApacheManager(temporary_file.name)
self.assertRaises(AddressDiscoveryError,
lambda: manager.listen_addresses)
# Test parsing of `Listen' directives with a port number but no IP address.
with tempfile.NamedTemporaryFile() as temporary_file:
with open(temporary_file.name, 'w') as handle:
handle.write('Listen 12345\n')
manager = ApacheManager(temporary_file.name)
assert any(a.port == 12345 for a in manager.listen_addresses)
# Test parsing of `Listen' directives with an IP address and port number.
with tempfile.NamedTemporaryFile() as temporary_file:
with open(temporary_file.name, 'w') as handle:
handle.write('Listen 127.0.0.2:54321\n')
manager = ApacheManager(temporary_file.name)
assert any(a.address == '127.0.0.2' and a.port == 54321
for a in manager.listen_addresses)
# Test that port discovery on the host system returns at least one port.
assert len(ApacheManager().listen_addresses) >= 1
def test_memory_usage(self):
"""Test that memory usage analysis works."""
manager = ApacheManager()
# Make sure there are Apache workers alive that have handled a couple of requests.
for i in range(10):
manager.fetch_status_page(manager.text_status_url)
assert sum(manager.memory_usage) > 0
# TODO Create a WSGI process group so we can perform a useful test here?
assert isinstance(manager.wsgi_process_groups, dict)
def test_memory_usage(self):
"""Test that memory usage analysis works."""
manager = ApacheManager()
# Make sure there are Apache workers alive that have handled a couple of requests.
for i in range(10):
manager.fetch_status_page(manager.text_status_url)
assert sum(manager.memory_usage) > 0
# TODO Create a WSGI process group so we can perform a useful test here?
assert isinstance(manager.wsgi_process_groups, dict)
def test_refresh(self):
"""Test refreshing of cached properties."""
manager = ApacheManager()
initial_accesses = manager.server_metrics['total_accesses']
time.sleep(1)
cached_accesses = manager.server_metrics['total_accesses']
assert cached_accesses == initial_accesses
manager.refresh()
time.sleep(1)
fresh_accesses = manager.server_metrics['total_accesses']
assert fresh_accesses > initial_accesses
def test_refresh(self):
"""Test refreshing of cached properties."""
manager = ApacheManager()
initial_uptime = manager.server_metrics['uptime']
time.sleep(1)
cached_uptime = manager.server_metrics['uptime']
assert cached_uptime == initial_uptime
manager.refresh()
time.sleep(1)
fresh_uptime = manager.server_metrics['uptime']
assert fresh_uptime > initial_uptime
def test_refresh(self):
"""Test refreshing of cached properties."""
manager = ApacheManager()
initial_uptime = manager.server_metrics['uptime']
time.sleep(1)
cached_uptime = manager.server_metrics['uptime']
assert cached_uptime == initial_uptime
manager.refresh()
time.sleep(1)
fresh_uptime = manager.server_metrics['uptime']
assert fresh_uptime > initial_uptime
def test_status_code_validation(self):
"""Test that unexpected HTTP responses from Apache raise an exception."""
manager = ApacheManager()
self.assertRaises(
StatusPageError,
manager.fetch_status_page,
manager.html_status_url + "-non-existing-endpoint",
)
def kill_idle_workers(dry_run):
time.sleep(1)
arguments = ['--max-memory-idle=1K']
if dry_run:
arguments.insert(0, '--dry-run')
workers_alive_before = set(w.pid for w in ApacheManager().workers
if w.is_alive)
exit_code, output = run_cli(arguments)
# Check that one or more worker processes were (simulated to be) killed.
assert exit_code == 0
assert re.search(r'Killing native worker \d+ \(idle\)', output)
# Check that one or more worker processes actually died?
if not dry_run:
workers_alive_after = set(w.pid
for w in ApacheManager().workers
if w.is_alive)
assert workers_alive_before != workers_alive_after
def __init__(self, name):
"""
Initialize a :class:`TemporaryWSGIApp` object.
:param name: The name of the WSGI application (a string).
"""
self.manager = ApacheManager()
self.name = 'apache-manager-%s' % name
def test_save_metrics(self):
"""Test that monitoring metrics can be saved to a text file."""
fd, temporary_file = tempfile.mkstemp()
try:
manager = ApacheManager()
manager.save_metrics(temporary_file)
with open(temporary_file) as handle:
keywords = [tokens[0] for tokens in (line.split() for line in handle) if tokens]
assert all(required_kw in keywords for required_kw in [
'busy-workers',
'bytes-per-request',
'bytes-per-second',
'cpu-load',
'idle-workers',
'requests-per-second',
'status-response',
'total-accesses',
'total-traffic',
'uptime',
'workers-killed-active',
'workers-killed-idle',
])
finally:
os.unlink(temporary_file)
def test_worker_table_parsing(self):
"""Test that parsing of worker information from the HTML status page works."""
manager = ApacheManager()
# We expect there to be at least one slot and worker.
assert manager.slots
assert manager.workers
# There will never be more workers than there are slots, although there
# may be more slots than workers. However we can't reasonable test this
# because it depends on the system's Apache configuration.
assert len(manager.slots) >= len(manager.workers)
retry(lambda: any(w.is_active for w in manager.workers))
retry(lambda: any(w.is_alive for w in manager.workers))
retry(lambda: any(w.is_idle for w in manager.workers))
retry(lambda: any(isinstance(w.acc, tuple) for w in manager.workers))
# Validate the WorkerStatus.acc property.
worker = manager.workers[0]
assert isinstance(worker.acc, tuple)
assert len(worker.acc) == 3
assert all(isinstance(n, int) for n in worker.acc)
# Validate the WorkerStatus.child property.
assert isinstance(worker.child, float)
# Validate the WorkerStatus.client property.
assert isinstance(worker.client, text_type) and worker.client
# Validate the WorkerStatus.conn property.
assert isinstance(worker.conn, float)
# Validate the WorkerStatus.cpu property.
assert isinstance(worker.cpu, float)
# Validate the WorkerStatus.req property.
assert isinstance(worker.req, int)
# Validate the WorkerStatus.slot property.
assert isinstance(worker.slot, float)
# Validate the WorkerStatus.srv property.
assert isinstance(worker.srv, tuple)
assert len(worker.srv) == 2
assert all(isinstance(n, int) for n in worker.srv)
# Validate the WorkerStatus.vhost property.
assert isinstance(worker.vhost, text_type) and worker.vhost
def main():
"""Command line interface for the ``apache-manager`` program."""
# Configure logging output.
coloredlogs.install(syslog=True)
# Command line option defaults.
data_file = '/tmp/apache-manager.txt'
dry_run = False
max_memory_active = None
max_memory_idle = None
max_ss = None
watch = False
zabbix_discovery = False
verbosity = 0
# Parse the command line options.
try:
options, arguments = getopt.getopt(sys.argv[1:], 'wa:i:t:f:znvqh', [
'watch', 'max-memory-active=', 'max-memory-idle=', 'max-ss=',
'max-time=', 'data-file=', 'zabbix-discovery', 'dry-run',
'simulate', 'verbose', 'quiet', 'help',
])
for option, value in options:
if option in ('-w', '--watch'):
watch = True
elif option in ('-a', '--max-memory-active'):
max_memory_active = parse_size(value)
elif option in ('-i', '--max-memory-idle'):
max_memory_idle = parse_size(value)
elif option in ('-t', '--max-ss', '--max-time'):
max_ss = parse_timespan(value)
elif option in ('-f', '--data-file'):
data_file = value
elif option in ('-z', '--zabbix-discovery'):
zabbix_discovery = True
elif option in ('-n', '--dry-run', '--simulate'):
logger.info("Performing a dry run ..")
dry_run = True
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
verbosity += 1
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
verbosity -= 1
elif option in ('-h', '--help'):
usage(__doc__)
return
except Exception as e:
sys.stderr.write("Error: %s!\n" % e)
sys.exit(1)
# Execute the requested action(s).
manager = ApacheManager()
try:
if max_memory_active or max_memory_idle or max_ss:
manager.kill_workers(
max_memory_active=max_memory_active,
max_memory_idle=max_memory_idle,
timeout=max_ss,
dry_run=dry_run,
)
elif watch and connected_to_terminal(sys.stdout):
watch_metrics(manager)
elif zabbix_discovery:
report_zabbix_discovery(manager)
elif data_file != '-' and verbosity >= 0:
for line in report_metrics(manager):
if line_is_heading(line):
line = ansi_wrap(line, color=HIGHLIGHT_COLOR)
print(line)
finally:
if (not watch) and (data_file == '-' or not dry_run):
manager.save_metrics(data_file)
def kill_timeout_worker():
manager = ApacheManager()
killed_processes = manager.kill_workers(timeout=30)
assert worker_pid in killed_processes
def kill_active_worker():
manager = ApacheManager()
killed_processes = manager.kill_workers(max_memory_active=1024*1024*50)
assert worker_pid in killed_processes
def test_text_status_page(self):
"""Test that the plain text status page can be fetched."""
manager = ApacheManager()
assert manager.text_status
def test_extract_metric(self):
"""Test that extract_metric() fails "gracefully" by returning a default value."""
manager = ApacheManager()
assert manager.extract_metric('This pattern is expected to never match', '42') == '42'
def kill_active_worker():
manager = ApacheManager()
killed_processes = manager.kill_workers(
max_memory_active=1024 * 1024 * 50)
assert worker_pid in killed_processes
def kill_timeout_worker():
manager = ApacheManager()
killed_processes = manager.kill_workers(timeout=30)
assert worker_pid in killed_processes
def test_html_status_page(self):
"""Test that the HTML status page can be fetched."""
manager = ApacheManager()
assert manager.html_status
def test_extract_metric(self):
"""Test that extract_metric() fails "gracefully" by returning a default value."""
manager = ApacheManager()
assert manager.extract_metric(
'This pattern is expected to never match', '42') == '42'
def main():
"""Command line interface for the ``apache-manager`` program."""
# Configure logging output.
coloredlogs.install()
# Command line option defaults.
data_file = '/tmp/apache-manager.txt'
dry_run = False
max_memory_active = None
max_memory_idle = None
max_ss = None
watch = False
zabbix_discovery = False
verbosity = 0
# Parse the command line options.
try:
options, arguments = getopt.getopt(sys.argv[1:], 'wa:i:t:f:znvqh', [
'watch', 'max-memory-active=', 'max-memory-idle=', 'max-ss=',
'max-time=', 'data-file=', 'zabbix-discovery', 'dry-run',
'simulate', 'verbose', 'quiet', 'help',
])
for option, value in options:
if option in ('-w', '--watch'):
watch = True
elif option in ('-a', '--max-memory-active'):
max_memory_active = parse_size(value)
elif option in ('-i', '--max-memory-idle'):
max_memory_idle = parse_size(value)
elif option in ('-t', '--max-ss', '--max-time'):
max_ss = parse_timespan(value)
elif option in ('-f', '--data-file'):
data_file = value
elif option in ('-z', '--zabbix-discovery'):
zabbix_discovery = True
elif option in ('-n', '--dry-run', '--simulate'):
logger.info("Performing a dry run ..")
dry_run = True
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
verbosity += 1
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
verbosity -= 1
elif option in ('-h', '--help'):
usage(__doc__)
return
except Exception as e:
sys.stderr.write("Error: %s!\n" % e)
sys.exit(1)
# Execute the requested action(s).
manager = ApacheManager()
try:
if max_memory_active or max_memory_idle or max_ss:
manager.kill_workers(
max_memory_active=max_memory_active,
max_memory_idle=max_memory_idle,
timeout=max_ss,
dry_run=dry_run,
)
if watch and connected_to_terminal(sys.stdout):
watch_metrics(manager)
elif zabbix_discovery:
report_zabbix_discovery(manager)
elif data_file != '-' and verbosity >= 0:
for line in report_metrics(manager):
if line_is_heading(line):
line = ansi_wrap(line, color=HIGHLIGHT_COLOR)
print(line)
finally:
if (not watch) and (data_file == '-' or not dry_run):
manager.save_metrics(data_file)
def test_server_metrics(self):
"""Test that server metrics parsing works."""
manager = ApacheManager()
assert manager.server_metrics['uptime'] > 0
def test_manager_metrics(self):
"""Test that the Apache manager successfully reports metrics about itself."""
manager = ApacheManager()
assert 'status_response' in manager.manager_metrics
assert manager.manager_metrics['workers_killed_active'] >= 0
assert manager.manager_metrics['workers_killed_idle'] >= 0