def restart_compute_service(extra_args=None):
extra_args = extra_args or []
test_config.compute_service.restart(extra_args=extra_args)
# Be absolutely certain the compute manager is ready before passing control
# back to caller.
poll_until(lambda: hosts_up("compute"), sleep_time=1, time_out=60)
wait_for_compute_service()
def test_confirm_ops_was_notified(self):
current_count = out_of_instance_memory_nofication_count()
# Additional ops notifications should have been added.
poll_until(out_of_instance_memory_nofication_count,
lambda count: original_notification_count < count,
sleep_time=1,
time_out=60)
def ensure_vz_actual_state(self, power_state):
"""Ensures the hypervisor has the correct state"""
def get_info_from_conn():
return self.conn.get_info(self.name)['state']
poll_until(get_info_from_conn,
lambda state: state == power_state,
sleep_time=2, time_out=60)
def test_get_init_pid(self):
def get_the_pid():
out, err = process("pgrep init | vzpid - | awk '/%s/{print $1}'"
% str(instance_info.local_id))
instance_info.pid = out.strip()
return len(instance_info.pid) > 0
poll_until(get_the_pid, sleep_time=10, time_out=60*10)
def find_evidence_scheduler_failed_in_logs():
"""Eavesdrop on the logs until we see the scheduler failed, or time-out."""
evidence = "Error scheduling " + initial_instance.name
poll_until(lambda: file(FLAGS.logfile, 'r').read(),
lambda log: evidence in log,
sleep_time=3,
time_out=60)
def wait_for_compute_service():
pid = test_config.compute_service.find_proc_id()
line = "Creating Consumer connection for Service compute from (pid=%d)" % pid
try:
poll_until(lambda: check_logs_for_message(line), sleep_time=1, time_out=60)
except exception.PollTimeOut:
raise RuntimeError("Could not find the line %s in the logs." % line)
def delete_test_entry(self):
fullname = TEST_NAME
self.driver.delete_entry(fullname, "A")
# It takes awhile for them to be deleted.
poll_until(lambda : self.driver.get_entries_by_name(TEST_NAME),
lambda list : len(list) == 0,
sleep_time=2, time_out=60)
def delete_test_entry(self):
fullname = TEST_NAME
self.driver.delete_entry(fullname, "A")
# It takes awhile for them to be deleted.
poll_until(lambda: self.driver.get_entries_by_name(TEST_NAME),
lambda list: len(list) == 0,
sleep_time=2,
time_out=60)
def wait_for_resize(self):
def is_finished_resizing():
instance = self.instance
if instance.status == "RESIZE":
return False
assert_equal("ACTIVE", instance.status)
return True
poll_until(is_finished_resizing, time_out = TIME_OUT_TIME)
def ensure_vz_actual_state(self, power_state):
"""Ensures the hypervisor has the correct state"""
def get_info_from_conn():
return self.conn.get_info(self.name)['state']
poll_until(get_info_from_conn,
lambda state: state == power_state,
sleep_time=2,
time_out=60)
def wait_for_successful_restart(self):
"""Wait until status becomes running."""
def is_finished_rebooting():
instance = self.instance
if instance.status == "REBOOT":
return False
assert_equal("ACTIVE", instance.status)
return True
poll_until(is_finished_rebooting, time_out = TIME_OUT_TIME)
def test_instance_created(self):
def check_status_of_instance():
status, err = process("sudo vzctl status %s | awk '{print $5}'"
% str(instance_info.local_id))
if string_in_list(status, ["running"]):
self.assertEqual("running", status.strip())
return True
else:
return False
poll_until(check_status_of_instance, sleep_time=5, time_out=60*8)
def update_and_wait_to_finish(self):
instance_info.dbaas_admin.management.update(instance_info.id)
def finished():
current_version = self.get_version()
if current_version == self.next_version:
return True
# The only valid thing for it to be aside from next_version is
# old version.
assert_equal(current_version, self.old_version)
poll_until(finished, sleep_time=1, time_out=3 * 60)
def wait_for_failure_status(self):
"""Wait until status becomes running."""
def is_finished_rebooting():
instance = self.instance
if instance.status == "REBOOT":
return False
assert_equal("SHUTDOWN", instance.status)
return True
poll_until(is_finished_rebooting, time_out = TIME_OUT_TIME)
def delete_all_entries(self):
"""Deletes all entries under the default domain."""
list = self.driver.get_entries()
for entry in list:
if entry.type == "A":
self.driver.delete_entry(name=entry.name, type=entry.type,
dns_zone=entry.dns_zone)
# It takes awhile for them to be deleted.
poll_until(lambda : self.driver.get_entries_by_name(TEST_NAME),
lambda list : len(list) == 0,
sleep_time=4, time_out=60)
def test_dns_entry_should_exist(self):
entry = instance_info.expected_dns_entry()
if entry:
def get_entries():
return dns_driver.get_entries_by_name(entry.name)
try:
poll_until(get_entries, lambda entries: len(entries) > 0,
sleep_time=2, time_out = 60)
except exception.PollTimeOut:
self.fail("Did not find name " + entry.name + \
" in the entries, which were as follows:"
+ str(dns_driver.get_entries()))
def _assert_volume_is_eventually_deleted(self, time_out=(3 * 60)):
"""Polls until some time_out to see if the volume is deleted.
This test is according to the database, not the REST API.
"""
def volume_not_found():
try:
self.db.volume_get(context.get_admin_context(), self.volume_id)
return False
except exception.VolumeNotFound:
return True
poll_until(volume_not_found, sleep_time=1, time_out=time_out)
def delete_all_entries(self):
"""Deletes all entries under the default domain."""
list = self.driver.get_entries()
for entry in list:
if entry.type == "A":
self.driver.delete_entry(name=entry.name,
type=entry.type,
dns_zone=entry.dns_zone)
# It takes awhile for them to be deleted.
poll_until(lambda: self.driver.get_entries_by_name(TEST_NAME),
lambda list: len(list) == 0,
sleep_time=4,
time_out=60)
def _assert_volume_is_eventually_deleted(self, time_out=(3 * 60)):
"""Polls until some time_out to see if the volume is deleted.
This test is according to the database, not the REST API.
"""
def volume_not_found():
try:
self.db.volume_get(context.get_admin_context(), self.volume_id)
return False
except exception.VolumeNotFound:
return True
poll_until(volume_not_found, sleep_time=1, time_out=time_out)
def test_api_get(self):
"""Wait until the volume is finished provisioning."""
volume = poll_until(lambda : self.story.get_volume(),
lambda volume : volume["status"] != "creating")
self.assertEqual(volume["status"], "available")
self.assert_volume_as_expected(volume)
self.assertTrue(volume["attach_status"], "detached")
def test_api_get(self):
"""Wait until the volume is finished provisioning."""
volume = poll_until(lambda: self.story.get_volume(),
lambda volume: volume["status"] != "creating")
self.assertEqual(volume["status"], "available")
self.assert_volume_as_expected(volume)
self.assertTrue(volume["attach_status"], "detached")
def wait_for_compute_host_up(self):
"""Wait for the compute host to appear as ready again.
If we don't do this, the scheduler will fail it.
"""
def ready():
results = dbapi.service_get_all_compute_memory(
context.get_admin_context())
for result in results:
(service, memory_mb) = result
needed_memory = memory_mb + 512
if needed_memory <= FLAGS.max_instance_memory_mb and \
Scheduler.service_is_up(service):
return True
return False
poll_until(ready, sleep_time=2, time_out=60)
def test_volume_resize_success(self):
def check_resize_status():
instance = instance_info.dbaas.instances.get(instance_info.id)
if instance.status == "ACTIVE":
return True
elif instance.status == "RESIZE":
return False
else:
fail("Status should not be %s" % instance.status)
poll_until(check_resize_status, sleep_time=2, time_out=300)
volumes = db.volume_get(context.get_admin_context(),
instance_info.volume_id)
assert_equal(volumes.status, 'in-use')
assert_equal(volumes.size, self.new_volume_size)
assert_equal(volumes.attach_status, 'attached')
def test_dns_entry_exist_should_be_removed_shortly_thereafter(self):
entry = instance_info.expected_dns_entry()
if not entry:
return
def get_entries():
return dns_driver.get_entries_by_name(entry.name)
try:
poll_until(get_entries, lambda entries : len(entries) == 0,
sleep_time=2, time_out=60)
except exception.PollTimeOut:
# Manually delete the rogue item
dns_driver.delete_entry(entry.name, entry.type, entry.dns_zone)
self.fail("The DNS entry was never deleted when the instance "
"was destroyed.")
def test_get_missing_volume(self):
try:
volume = poll_until(lambda : self.story.api.get(self.story.context,
self.story.volume_id),
lambda volume : volume["status"] != "deleted")
self.assertEqual(volume["deleted"], False)
except exception.VolumeNotFound:
pass
def test_get_missing_volume(self):
try:
volume = poll_until(lambda: self.story.api.get(self.story.context,
self.story.volume_id),
lambda volume: volume["status"] != "deleted")
self.assertEqual(volume["deleted"], False)
except exception.VolumeNotFound:
pass
def test_resize(self):
self.story.api.resize(self.story.context, self.story.volume_id,
self.story.resize_volume_size)
volume = poll_until(lambda : self.story.get_volume(),
lambda volume : volume["status"] == "resized")
self.assertEqual(volume["status"], "resized")
self.assertTrue(volume["attach_status"], "attached")
self.assertTrue(volume['size'], self.story.resize_volume_size)
def test_resize(self):
self.story.api.resize(self.story.context, self.story.volume_id,
self.story.resize_volume_size)
volume = poll_until(lambda: self.story.get_volume(),
lambda volume: volume["status"] == "resized")
self.assertEqual(volume["status"], "resized")
self.assertTrue(volume["attach_status"], "attached")
self.assertTrue(volume['size'], self.story.resize_volume_size)
def test_instance_wait_for_initialize_guest_to_exit_polling(self):
def compute_manager_finished():
return util.check_logs_for_message("INFO reddwarf.compute.manager [-] Guest is now running on instance %s"
% str(instance_info.local_id))
poll_until(compute_manager_finished, sleep_time=2, time_out=60)
def wait_for_rest_api_to_show_status_as_failed(self, time_out):
"""Confirms the REST API state becomes failure."""
poll_until(self._get_status_tuple, self._assert_status_failure,
sleep_time=1, time_out=time_out)
def test_api_get(self):
"""Wait until the volume is a FAILURE."""
volume = poll_until(lambda : self.storyFail.get_volume(),
lambda volume : volume["status"] != "creating")
self.assertEqual(volume["status"], "error")
self.assertTrue(volume["attach_status"], "detached")
def test_volume_detached(self):
poll_until(self._check_volume_detached, sleep_time=1, time_out=3 * 60)
def wait_for_compute_instance_to_suspend(self):
"""Polls until the compute instance is known to be suspended."""
poll_until(self._get_compute_instance_state,
lambda state: state in VALID_ABORT_STATES,
sleep_time=1,
time_out=FLAGS.reddwarf_instance_suspend_time_out)
def wait_for_rest_api_to_show_status_as_failed(self, time_out):
"""Confirms the REST API state becomes failure."""
poll_until(self._get_status_tuple,
self._assert_status_failure,
sleep_time=1,
time_out=time_out)
def test_api_get(self):
"""Wait until the volume is a FAILURE."""
volume = poll_until(lambda: self.storyFail.get_volume(),
lambda volume: volume["status"] != "creating")
self.assertEqual(volume["status"], "error")
self.assertTrue(volume["attach_status"], "detached")
def ensure_vz_power_state(self, power_state):
"""Ensures the database has the correct state"""
poll_until(self._get_compute_instance_state,
lambda state: state == power_state,
sleep_time=2, time_out=60)
def ensure_vz_power_state(self, power_state):
"""Ensures the database has the correct state"""
poll_until(self._get_compute_instance_state,
lambda state: state == power_state,
sleep_time=2,
time_out=60)
def find_evidence_scheduler_failed_in_logs():
"""Eavesdrop on the logs until we see the scheduler failed, or time-out."""
evidence = "Error scheduling " + initial_instance.name
poll_until(lambda: file(FLAGS.logfile, 'r').read(),
lambda log: evidence in log, sleep_time=3, time_out=60)
def test_confirm_ops_was_notified(self):
current_count = out_of_instance_memory_nofication_count()
# Additional ops notifications should have been added.
poll_until(out_of_instance_memory_nofication_count,
lambda count: original_notification_count < count,
sleep_time=1, time_out=60)
def wait_for_compute_instance_to_suspend(self):
"""Polls until the compute instance is known to be suspended."""
poll_until(self._get_compute_instance_state,
lambda state: state in VALID_ABORT_STATES,
sleep_time=1,
time_out=FLAGS.reddwarf_instance_suspend_time_out)
def wait_for_broken_connection(self):
"""Wait until our connection breaks."""
poll_until(self.connection.is_connected,
lambda connected : not connected, time_out = TIME_OUT_TIME)
