def setUp(self):
if os.path.exists("%sglance-registry" % glance_bin_root()):
reg_path = "%sglance-registry" % glance_bin_root()
else:
reg_path = "/usr/bin/glance-registry"
self.service = Service(python_cmd_list() +
[reg_path, glance_reg_conf()])
class Daemon(object):
"""Starts a daemon."""
def __init__(self, alternate_path=None, conf_file_name=None,
extra_cmds=None, service_path_root=None, service_path=None):
# The path to the daemon bin if the other one doesn't work.
self.alternate_path = alternate_path
self.extra_cmds = extra_cmds or []
# The name of a test config value which points to a conf file.
self.conf_file_name = conf_file_name
# The name of a test config value, which is inserted into the service_path.
self.service_path_root = service_path_root
# The first path to the daemon bin we try.
self.service_path = service_path or "%s"
def run(self):
# Print out everything to make it
print("Looking for config value %s..." % self.service_path_root)
print(CONFIG.values[self.service_path_root])
path = self.service_path % CONFIG.values[self.service_path_root]
print("Path = %s" % path)
if not os.path.exists(path):
path = self.alternate_path
if path is None:
fail("Could not find path to %s" % self.service_path_root)
conf_path = str(CONFIG.values[self.conf_file_name])
cmds = CONFIG.python_cmd_list() + [path] + self.extra_cmds + \
[conf_path]
print("Running cmds: %s" % cmds)
self.service = Service(cmds)
if not self.service.is_service_alive():
self.service.start()
class Daemon(object):
"""Starts a daemon."""
def __init__(self, alternate_path=None, conf_file_name=None,
extra_cmds=None, service_path_root=None, service_path=None):
# The path to the daemon bin if the other one doesn't work.
self.alternate_path = alternate_path
self.extra_cmds = extra_cmds or []
# The name of a test config value which points to a conf file.
self.conf_file_name = conf_file_name
# The name of a test config value, which is inserted into the service_path.
self.service_path_root = service_path_root
# The first path to the daemon bin we try.
self.service_path = service_path or "%s"
def run(self):
# Print out everything to make it
print("Looking for config value %s..." % self.service_path_root)
print(CONFIG.values[self.service_path_root])
path = self.service_path % CONFIG.values[self.service_path_root]
print("Path = %s" % path)
if not os.path.exists(path):
path = self.alternate_path
if path is None:
fail("Could not find path to %s" % self.service_path_root)
conf_path = str(CONFIG.values[self.conf_file_name])
cmds = CONFIG.python_cmd_list() + [path] + self.extra_cmds + \
[conf_path]
print("Running cmds: %s" % cmds)
self.service = Service(cmds)
if not self.service.is_service_alive():
self.service.start()
class KeystoneAll(unittest.TestCase):
"""Starts the Keystone combined daemon."""
def setUp(self):
path = keystone_bin("keystone-all")
self.service = Service(python_cmd_list() + [path, "-c %s" % keystone_conf()])
def test_start(self):
if not self.service.is_service_alive():
self.service.start()
class KeystoneAll(unittest.TestCase):
"""Starts the Keystone combined daemon."""
def setUp(self):
path = keystone_bin("keystone-all")
self.service = Service(python_cmd_list() +
[path, "-c %s" % keystone_conf()])
def test_start(self):
if not self.service.is_service_alive():
self.service.start()
class Scheduler(unittest.TestCase):
"""Starts the Scheduler Service."""
def setUp(self):
self.service = Service(
python_cmd_list() + ["%s/bin/nova-scheduler" % nova_code_root(), "--flagfile=%s" % nova_conf()]
)
def test_start(self):
if not either_web_service_is_up():
self.service.start()
class Reaper(unittest.TestCase):
"""Starts the Reaper."""
def setUp(self):
self.service = Service(
python_cmd_list() + ["%s/bin/nova-reaper" % nova_code_root(), "--flagfile=%s" % nova_conf()]
)
def test_start(self):
if not self.service.is_service_alive():
self.service.start()
class Reaper(unittest.TestCase):
"""Starts the Reaper."""
def setUp(self):
self.service = Service(python_cmd_list() +
["%s/bin/nova-reaper" % nova_code_root(),
"--flagfile=%s" % nova_conf()])
def test_start(self):
if not self.service.is_service_alive():
self.service.start()
def check_agent_path_is_correct(self):
"""Make sure the agent binary listed in the config is correct."""
self.agent_bin = str(test_config.values["agent_bin"])
nova_conf = str(test_config.values["nova_conf"])
assert_true(path.exists(self.agent_bin),
"Agent not found at path: %s" % self.agent_bin)
self.agent = Service(cmd=[
self.agent_bin,
"--flagfile=%s" % nova_conf, "--rabbit_reconnect_wait_time=1",
"--preset_instance_id=-99"
])
class Scheduler(unittest.TestCase):
"""Starts the Scheduler Service."""
def setUp(self):
self.service = Service(python_cmd_list() +
["%s/bin/nova-scheduler" % nova_code_root(),
"--flagfile=%s" % nova_conf()])
def test_start(self):
if not either_web_service_is_up():
self.service.start()
class GlanceApi(unittest.TestCase):
"""Starts the Glance API."""
def setUp(self):
if os.path.exists("%sglance-api" % glance_bin_root()):
reg_path = "%sglance-api" % glance_bin_root()
else:
reg_path = "/usr/bin/glance-api"
self.service = Service(python_cmd_list() + [reg_path, glance_api_conf()])
def test_start(self):
if not either_web_service_is_up():
self.service.start()
class GlanceRegistry(unittest.TestCase):
"""Starts the Glance Registry."""
def setUp(self):
if os.path.exists("%sglance-registry" % glance_bin_root()):
reg_path = "%sglance-registry" % glance_bin_root()
else:
reg_path = "/usr/bin/glance-registry"
self.service = Service(python_cmd_list() +
[reg_path, glance_reg_conf()])
def test_start(self):
if not either_web_service_is_up():
self.service.start()
def setUp(self):
if os.path.exists("%sglance-registry" % glance_bin_root()):
reg_path = "%sglance-registry" % glance_bin_root()
else:
reg_path = "/usr/bin/glance-registry"
self.service = Service(python_cmd_list() +
[reg_path, glance_reg_conf()])
def run(self):
# Print out everything to make it
print("Looking for config value %s..." % self.service_path_root)
print(CONFIG.values[self.service_path_root])
path = self.service_path % CONFIG.values[self.service_path_root]
print("Path = %s" % path)
if not os.path.exists(path):
path = self.alternate_path
if path is None:
fail("Could not find path to %s" % self.service_path_root)
conf_path = str(CONFIG.values[self.conf_file_name])
cmds = CONFIG.python_cmd_list() + [path] + self.extra_cmds + \
[conf_path]
print("Running cmds: %s" % cmds)
self.service = Service(cmds)
if not self.service.is_service_alive():
self.service.start()
def check_agent_path_is_correct(self):
"""Make sure the agent binary listed in the config is correct."""
self.agent_bin = str(test_config.values["agent_bin"])
nova_conf = str(test_config.values["nova_conf"])
assert_true(path.exists(self.agent_bin),
"Agent not found at path: %s" % self.agent_bin)
self.agent = Service(cmd=[self.agent_bin, "--flagfile=%s" % nova_conf,
"--rabbit_reconnect_wait_time=1",
"--preset_instance_id=-99"])
def run(self):
# Print out everything to make it
print("Looking for config value %s..." % self.service_path_root)
print(CONFIG.values[self.service_path_root])
path = self.service_path % CONFIG.values[self.service_path_root]
print("Path = %s" % path)
if not os.path.exists(path):
path = self.alternate_path
if path is None:
fail("Could not find path to %s" % self.service_path_root)
conf_path = str(CONFIG.values[self.conf_file_name])
cmds = CONFIG.python_cmd_list() + [path] + self.extra_cmds + \
[conf_path]
print("Running cmds: %s" % cmds)
self.service = Service(cmds)
if not self.service.is_service_alive():
self.service.start()
def _setup():
"""Initializes the module."""
from tests.util.users import Users
global clean_slate
global compute_service
global dbaas
global dbaas_image
global dbaas_url
global glance_image
global keystone_service
global nova
global nova_auth_url
global reddwarf_auth_url
global test_mgmt
global typical_nova_image_name
global use_reaper
global use_venv
global users
global values
global version_url
global volume_service
global white_box
global instance_create_time
clean_slate = os.environ.get("CLEAN_SLATE", "False") == "True"
values = load_configuration()
if os.environ.get("FAKE_MODE", "False") == "True":
values["fake_mode"] = True
use_venv = values.get("use_venv", True)
nova_auth_url = str(
values.get("nova_auth_url", "http://localhost:5000/v2.0"))
reddwarf_auth_url = str(
values.get("reddwarf_auth_url", "http://localhost:5000/v1.1"))
dbaas_url = str(values.get("dbaas_url",
"http://localhost:8775/v1.0/dbaas"))
version_url = str(values.get("version_url", "http://localhost:8775/"))
nova_url = str(values.get("nova_url", "http://localhost:8774/v1.1"))
nova_code_root = str(values["nova_code_root"])
nova_conf = str(values["nova_conf"])
keystone_conf = str(values["keystone_conf"])
reddwarf_code_root = str(values["reddwarf_code_root"])
reddwarf_conf = str(values["reddwarf_conf"])
glance_image = str(values["glance_image"])
use_reaper = values["use_reaper"]
instance_create_time = values.get('instance_create_time', 16 * 60)
if not nova_conf:
raise ValueError("Configuration value \"nova_conf\" not found.")
if str(values.get("reddwarf_api_format", "new")) == "old":
dbaas = WebService(cmd=python_cmd_list() + [
"%s/bin/reddwarf-api" % reddwarf_code_root,
"--flagfile=%s" % reddwarf_conf
],
url=dbaas_url)
else:
dbaas = WebService(cmd=python_cmd_list() + [
"%s/bin/reddwarf-server" % reddwarf_code_root,
"--config-file=%s" % reddwarf_conf
],
url=dbaas_url)
nova = WebService(
cmd=python_cmd_list() +
["%s/bin/nova-api" % nova_code_root,
"--flagfile=%s" % nova_conf],
url=nova_url)
volume_service = Service(
cmd=python_cmd_list() +
["%s/bin/nova-volume" % nova_code_root,
"--flagfile=%s" % nova_conf])
compute_service = Service(
cmd=python_cmd_list() +
["%s/bin/nova-compute" % nova_code_root,
"--flagfile=%s" % nova_conf])
keystone_service = Service(
python_cmd_list() +
[keystone_bin("keystone-auth"),
"-c %s" % keystone_conf])
users = Users(values["users"])
dbaas_image = values.get("dbaas_image", None)
typical_nova_image_name = values.get("typical_nova_image_name", None)
# If true, we import certain classes and test using internal code.
white_box = values.get("white_box", False)
# If true, we run the mgmt tests, if not we don't.
test_mgmt = values.get("test_mgmt", False)
def setUp(self):
path = keystone_bin("keystone-all")
self.service = Service(python_cmd_list() +
[path, "-c %s" % keystone_conf()])
def setUp(self):
self.service = Service(
python_cmd_list() + ["%s/bin/nova-reaper" % nova_code_root(), "--flagfile=%s" % nova_conf()]
)
def setUp(self):
self.service = Service(python_cmd_list() +
["%s/bin/nova-reaper" % nova_code_root(),
"--flagfile=%s" % nova_conf()])
def setUp(self):
path = keystone_bin("keystone-all")
self.service = Service(python_cmd_list() + [path, "-c %s" % keystone_conf()])
class WhenAgentRunsAsRabbitGoesUpAndDown(object):
"""Tests the agent is ok when Rabbit
"""
def __init__(self):
self.rabbit = Rabbit()
self.send_after_reconnect_errors = 0
self.tolerated_send_errors = 0
@after_class
def stop_agent(self):
self.agent.stop()
def _send(self):
original_queue_count = self.rabbit.get_queue_items()
@time_out(5)
def send_msg_with_timeout():
self.rabbit.declare_queue(topic_name())
version = rpc.call(context.get_admin_context(),
topic_name(),
{"method": "version",
"args": {"package_name": "dpkg"}
})
return { "status":"good", "version": version }
try:
return send_msg_with_timeout()
except Exception as e:
# If the Python side works, we should at least see an item waiting
# in the queue.
# Whether we see this determines if the failure to send is Nova's
# fault or Sneaky Petes.
print("Error making RPC call: %s" % e)
print("Original queue count = %d, "
"current count = %d" % (original_queue_count,
self.rabbit.get_queue_items()))
# In the Kombu driver there is a bug where after restarting rabbit
# the first message to be sent fails with a broken pipe. So here we
# tolerate one such bug but no more.
if not isinstance(e, TimeoutError):
self.send_after_reconnect_errors += 1
if self.send_after_reconnect_errors > self.tolerated_send_errors:
fail("Exception while making RPC call: %s" % e)
if self.rabbit.get_queue_items() > original_queue_count:
return { "status":"bad", "blame":"agent"}
else:
return { "status":"bad", "blame":"host"}
def _send_allow_for_host_bug(self):
while True:
result = self._send()
if result['status'] == "good":
return result["version"]
else:
if result['blame'] == "agent":
fail("Nova Host put a message on the queue but the agent "
"never responded.")
@test
def check_agent_path_is_correct(self):
"""Make sure the agent binary listed in the config is correct."""
self.agent_bin = str(test_config.values["agent_bin"])
nova_conf = str(test_config.values["nova_conf"])
assert_true(path.exists(self.agent_bin),
"Agent not found at path: %s" % self.agent_bin)
self.agent = Service(cmd=[self.agent_bin, "--flagfile=%s" % nova_conf,
"--rabbit_reconnect_wait_time=1",
"--preset_instance_id=-99"])
@test(depends_on=[check_agent_path_is_correct])
@time_out(60)
def send_agent_a_message(self):
self.rabbit.start()
self.agent.start(time_out=30)
result = self._send()
print("RESULT:%s" % result)
assert_equal(result['status'], "good")
@test(depends_on=[send_agent_a_message])
@time_out(30)
def make_sure_we_can_identify_an_agent_failure(self):
# This is so confusing, but it has to be, so listen up:
# Nova code has issues sending messages so we either don't test this
# or make allowances for Kombu's bad behavior. This test runs before
# we start the agent and makes sure if Nova successfully sends a
# message and the agent never answers it this test can identify that
# and fail.
self.agent.stop()
result = self._send()
assert_equal(result['status'], 'bad')
assert_equal(result['blame'], 'agent')
@test(depends_on=[make_sure_we_can_identify_an_agent_failure])
def stop_rabbit(self):
if self.rabbit.is_alive:
self.rabbit.stop()
assert_false(self.rabbit.is_alive)
self.rabbit.reset()
@test(depends_on=[check_agent_path_is_correct, stop_rabbit])
def start_agent_while_rabbit_is_off(self):
"""Starts the agent as rabbit is stopped.
Checks to make sure the agent doesn't just give up if it can't connect
to Rabbit, and also that the memory doesn't grow as it increasingly
creates connections.
"""
self.agent.start()
mem = self.agent.get_memory_info()
self.original_mapped = mem.mapped
@test(depends_on=[start_agent_while_rabbit_is_off])
def memory_should_not_increase_as_amqp_login_fails(self):
"""The agent should not spend memory on failed connections."""
#TODO(tim.simpson): This operates on the assumption that the the agent
# will try to reconnect multiple times while we sleep.
# Explanation: the syslog (where the agent logs now reside) is
# unreadable by the test user, so we can't count the original
# failures and wait until we know the agent has tried to reconnect
# several times before testing again. Instead we just sleep.
# Once we log to a readable file we should fix that.
#self.original_fail_count = count_message_occurrence_in_file(
# "/var/log/syslog", "Error establishing AMQP connection"
#)
# Memory should not go up as the connection fails.
print("Original mapped memory : %d" % self.original_mapped)
# I've noticed that the memory jumps up a bit between 5 and 10 seconds
# after it starts and then holds steady. So instead of taking the
# original count, let's wait a bit and use that.
time.sleep(10)
self.original_mapped = self.agent.get_memory_info().mapped
print("Mapped memory at 10 seconds : %d" % self.original_mapped)
total_seconds = 0
mapped = []
for i in range(4):
time.sleep(5)
total_seconds += 5
mapped.append(self.agent.get_memory_info().mapped)
print("Mapped memory after %d seconds : %d"
% (total_seconds, mapped[-1]))
if self.original_mapped < mapped[-1]:
fail("Oh no, after %d seconds memory rose from %d to %d!"
% (total_seconds, self.original_mapped, mapped[-1]))
if mapped[-1] > 30 * 1024:
fail("Whoa, why is mapped memory = %d for procid=%d, proc= %s?"
% (mapped[-1], self.agent.find_proc_id(), self.agent_bin))
@test(depends_on=[memory_should_not_increase_as_amqp_login_fails])
def start_rabbit(self):
"""Start rabbit."""
self.rabbit.start()
assert_true(self.rabbit.is_alive)
@test(depends_on=[start_rabbit])
@time_out(30)
def send_message(self):
"""Tests that the agent auto-connects to rabbit and gets a message."""
version = self._send_allow_for_host_bug()
assert_true(version is not None)
matches = re.search("(\\w+)\\.(\\w+)\\.(\\w+)\\.(\\w+)", version)
assert_true(matches is not None)
@test(depends_on=[send_message])
def restart_rabbit_again(self):
"""Now stop and start rabbit, ensuring the agent reconnects."""
self.rabbit.stop()
assert_false(self.rabbit.is_alive)
self.rabbit.reset()
self.rabbit.start()
assert_true(self.rabbit.is_alive)
@test(depends_on=[restart_rabbit_again])
@time_out(30)
def send_message_again_1(self):
"""Sends a message."""
self.tolerated_send_errors = 1
self.send_message()
@test(depends_on=[send_message_again_1])
@time_out(30)
def send_message_again_2a(self):
"""The agent should be able to receive messages after reconnecting."""
self.send_message()
@test(depends_on=[send_message_again_1])
@time_out(30)
def send_message_again_2b(self):
"""The agent should be able to receive messages after reconnecting."""
self.send_message()
class WhenAgentRunsAsRabbitGoesUpAndDown(object):
"""Tests the agent is ok when Rabbit
"""
def __init__(self):
self.rabbit = Rabbit()
self.send_after_reconnect_errors = 0
self.tolerated_send_errors = 0
@after_class
def stop_agent(self):
self.agent.stop()
def _send(self):
original_queue_count = self.rabbit.get_queue_items()
@time_out(5)
def send_msg_with_timeout():
self.rabbit.declare_queue(topic_name())
version = rpc.call(context.get_admin_context(), topic_name(), {
"method": "version",
"args": {
"package_name": "dpkg"
}
})
return {"status": "good", "version": version}
try:
return send_msg_with_timeout()
except Exception as e:
# If the Python side works, we should at least see an item waiting
# in the queue.
# Whether we see this determines if the failure to send is Nova's
# fault or Sneaky Petes.
print("Error making RPC call: %s" % e)
print("Original queue count = %d, "
"current count = %d" %
(original_queue_count, self.rabbit.get_queue_items()))
# In the Kombu driver there is a bug where after restarting rabbit
# the first message to be sent fails with a broken pipe. So here we
# tolerate one such bug but no more.
if not isinstance(e, TimeoutError):
self.send_after_reconnect_errors += 1
errors = self.send_after_reconnect_errors
if errors > self.tolerated_send_errors:
fail("Exception while making RPC call: %s" % e)
if self.rabbit.get_queue_items() > original_queue_count:
return {"status": "bad", "blame": "agent"}
else:
return {"status": "bad", "blame": "host"}
def _send_allow_for_host_bug(self):
while True:
result = self._send()
if result['status'] == "good":
return result["version"]
else:
if result['blame'] == "agent":
fail("Nova Host put a message on the queue but the agent "
"never responded.")
@test
def check_agent_path_is_correct(self):
"""Make sure the agent binary listed in the config is correct."""
self.agent_bin = str(test_config.values["agent_bin"])
nova_conf = str(test_config.values["nova_conf"])
assert_true(path.exists(self.agent_bin),
"Agent not found at path: %s" % self.agent_bin)
self.agent = Service(cmd=[
self.agent_bin,
"--flagfile=%s" % nova_conf, "--rabbit_reconnect_wait_time=1",
"--preset_instance_id=-99"
])
@test(depends_on=[check_agent_path_is_correct])
@time_out(60)
def send_agent_a_message(self):
self.rabbit.start()
self.agent.start(time_out=30)
result = self._send()
print("RESULT:%s" % result)
assert_equal(result['status'], "good")
@test(depends_on=[send_agent_a_message])
@time_out(30)
def make_sure_we_can_identify_an_agent_failure(self):
# This is so confusing, but it has to be, so listen up:
# Nova code has issues sending messages so we either don't test this
# or make allowances for Kombu's bad behavior. This test runs before
# we start the agent and makes sure if Nova successfully sends a
# message and the agent never answers it this test can identify that
# and fail.
self.agent.stop()
result = self._send()
assert_equal(result['status'], 'bad')
assert_equal(result['blame'], 'agent')
@test(depends_on=[make_sure_we_can_identify_an_agent_failure])
def stop_rabbit(self):
if self.rabbit.is_alive:
self.rabbit.stop()
assert_false(self.rabbit.is_alive)
self.rabbit.reset()
@test(depends_on=[check_agent_path_is_correct, stop_rabbit])
def start_agent_while_rabbit_is_off(self):
"""Starts the agent as rabbit is stopped.
Checks to make sure the agent doesn't just give up if it can't connect
to Rabbit, and also that the memory doesn't grow as it increasingly
creates connections.
"""
self.agent.start()
mem = self.agent.get_memory_info()
self.original_mapped = mem.mapped
@test(depends_on=[start_agent_while_rabbit_is_off])
def memory_should_not_increase_as_amqp_login_fails(self):
"""The agent should not spend memory on failed connections."""
#TODO(tim.simpson): This operates on the assumption that the the agent
# will try to reconnect multiple times while we sleep.
# Explanation: the syslog (where the agent logs now reside) is
# unreadable by the test user, so we can't count the original
# failures and wait until we know the agent has tried to reconnect
# several times before testing again. Instead we just sleep.
# Once we log to a readable file we should fix that.
#self.original_fail_count = count_message_occurrence_in_file(
# "/var/log/syslog", "Error establishing AMQP connection"
#)
# Memory should not go up as the connection fails.
print("Original mapped memory : %d" % self.original_mapped)
# I've noticed that the memory jumps up a bit between 5 and 10 seconds
# after it starts and then holds steady. So instead of taking the
# original count, let's wait a bit and use that.
time.sleep(10)
self.original_mapped = self.agent.get_memory_info().mapped
print("Mapped memory at 10 seconds : %d" % self.original_mapped)
total_seconds = 0
mapped = []
for i in range(4):
time.sleep(5)
total_seconds += 5
mapped.append(self.agent.get_memory_info().mapped)
print("Mapped memory after %d seconds : %d" %
(total_seconds, mapped[-1]))
if self.original_mapped < mapped[-1]:
fail("Oh no, after %d seconds memory rose from %d to %d!" %
(total_seconds, self.original_mapped, mapped[-1]))
if mapped[-1] > 30 * 1024:
fail("Whoa, why is mapped memory = %d for procid=%d, proc= %s?" %
(mapped[-1], self.agent.find_proc_id(), self.agent_bin))
@test(depends_on=[memory_should_not_increase_as_amqp_login_fails])
def start_rabbit(self):
"""Start rabbit."""
self.rabbit.start()
assert_true(self.rabbit.is_alive)
@test(depends_on=[start_rabbit])
@time_out(30)
def send_message(self):
"""Tests that the agent auto-connects to rabbit and gets a message."""
version = self._send_allow_for_host_bug()
assert_true(version is not None)
matches = re.search("(\\w+)\\.(\\w+)\\.(\\w+)\\.(\\w+)", version)
assert_true(matches is not None)
@test(depends_on=[send_message])
def restart_rabbit_again(self):
"""Now stop and start rabbit, ensuring the agent reconnects."""
self.rabbit.stop()
assert_false(self.rabbit.is_alive)
self.rabbit.reset()
self.rabbit.start()
assert_true(self.rabbit.is_alive)
@test(depends_on=[restart_rabbit_again])
@time_out(30)
def send_message_again_1(self):
"""Sends a message."""
self.tolerated_send_errors = 1
self.send_message()
@test(depends_on=[send_message_again_1])
@time_out(30)
def send_message_again_2a(self):
"""The agent should be able to receive messages after reconnecting."""
self.send_message()
@test(depends_on=[send_message_again_1])
@time_out(30)
def send_message_again_2b(self):
"""The agent should be able to receive messages after reconnecting."""
self.send_message()