def wait_for_compute_service():
pid = test_config.compute_service.find_proc_id()
line = "Creating Consumer connection for Service compute from (pid=%d)" % pid
try:
poll_until(lambda: check_logs_for_message(line), sleep_time=1, time_out=60)
except exception.PollTimeOut:
raise RuntimeError("Could not find the line %s in the logs." % line)
def test_get_init_pid(self):
def get_the_pid():
out, err = process("pgrep init | vzpid - | awk '/%s/{print $1}'"
% str(instance_info.local_id))
instance_info.pid = out.strip()
return len(instance_info.pid) > 0
poll_until(get_the_pid, sleep_time=10, time_out=(60 * 10))
def restart_compute_service(extra_args=None):
extra_args = extra_args or []
test_config.compute_service.restart(extra_args=extra_args)
# Be absolutely certain the compute manager is ready before passing control
# back to caller.
poll_until(lambda: hosts_up("compute"), sleep_time=1, time_out=60)
wait_for_compute_service()
def wait_for_broken_connection(self):
"""Wait until our connection breaks."""
if not USE_IP:
return
poll_until(self.connection.is_connected,
lambda connected: not connected,
time_out=TIME_OUT_TIME)
def wait_for_resize(self):
def is_finished_resizing():
instance = self.instance
if instance.status == "RESIZE":
return False
assert_equal("ACTIVE", instance.status)
return True
poll_until(is_finished_resizing, time_out=TIME_OUT_TIME)
def wait_for_resize(self):
def is_finished_resizing():
instance = self.instance
if instance.status == "RESIZE":
return False
assert_equal("ACTIVE", instance.status)
return True
poll_until(is_finished_resizing, time_out=TIME_OUT_TIME)
def wait_for_broken_connection(self):
"""Wait until our connection breaks."""
if not USE_IP:
return
if not hasattr(self, "connection"):
return
poll_until(self.connection.is_connected,
lambda connected: not connected,
time_out=TIME_OUT_TIME)
def wait_for_failure_status(self):
"""Wait until status becomes running."""
def is_finished_rebooting():
instance = self.instance
if instance.status == "REBOOT":
return False
assert_equal("SHUTDOWN", instance.status)
return True
poll_until(is_finished_rebooting, time_out=TIME_OUT_TIME)
def test_instance_created(self):
def check_status_of_instance():
status, err = process("sudo vzctl status %s | awk '{print $5}'"
% str(instance_info.local_id))
if string_in_list(status, ["running"]):
self.assertEqual("running", status.strip())
return True
else:
return False
poll_until(check_status_of_instance, sleep_time=5, time_out=(60 * 8))
def wait_for_successful_restart(self):
"""Wait until status becomes running."""
def is_finished_rebooting():
instance = self.instance
if instance.status == "REBOOT":
return False
assert_equal("ACTIVE", instance.status)
return True
poll_until(is_finished_rebooting, time_out=TIME_OUT_TIME)
def wait_for_compute_service():
pid = test_config.compute_service.find_proc_id()
line = "Creating Consumer connection for Service compute from (pid=%d)" % \
pid
try:
poll_until(lambda: check_logs_for_message(line),
sleep_time=1,
time_out=60)
except exception.PollTimeOut:
raise RuntimeError("Could not find the line %s in the logs." % line)
def wait_for_successful_restart(self):
"""Wait until status becomes running."""
def is_finished_rebooting():
instance = self.instance
if instance.status == "REBOOT":
return False
assert_equal("ACTIVE", instance.status)
return True
poll_until(is_finished_rebooting, time_out=TIME_OUT_TIME)
def test_backup_created(self):
# This version just checks the REST API status.
def result_is_active():
backup = instance_info.dbaas.backups.get(backup_info.id)
if backup.status == "COMPLETED":
return True
else:
assert_not_equal("FAILED", backup.status)
return False
poll_until(result_is_active)
def update_and_wait_to_finish(self):
instance_info.dbaas_admin.management.update(instance_info.id)
def finished():
current_version = self.get_version()
if current_version == self.next_version:
return True
# The only valid thing for it to be aside from next_version is
# old version.
assert_equal(current_version, self.old_version)
poll_until(finished, sleep_time=1, time_out=3 * 60)
def test_backup_created(self):
# This version just checks the REST API status.
def result_is_active():
backup = instance_info.dbaas.backups.get(backup_info.id)
if backup.status == "COMPLETED":
return True
else:
assert_not_equal("FAILED", backup.status)
return False
poll_until(result_is_active)
def update_and_wait_to_finish(self):
instance_info.dbaas_admin.management.update(instance_info.id)
def finished():
current_version = self.get_version()
if current_version == self.next_version:
return True
# The only valid thing for it to be aside from next_version is
# old version.
assert_equal(current_version, self.old_version)
poll_until(finished, sleep_time=1, time_out=3 * 60)
def wait_for_failure_status(self):
"""Wait until status becomes running."""
def is_finished_rebooting():
instance = self.instance
if instance.status == "REBOOT" or instance.status == "ACTIVE":
return False
# The reason we check for BLOCKED as well as SHUTDOWN is because
# Upstart might try to bring mysql back up after the borked
# connection and the guest status can be either
assert_true(instance.status in ("SHUTDOWN", "BLOCKED"))
return True
poll_until(is_finished_rebooting, time_out=TIME_OUT_TIME)
def test_volume_resize_success(self):
def check_resize_status():
instance = instance_info.dbaas.instances.get(instance_info.id)
if instance.status == "ACTIVE":
return True
elif instance.status == "RESIZE":
return False
else:
fail("Status should not be %s" % instance.status)
poll_until(check_resize_status, sleep_time=2, time_out=300)
instance = instance_info.dbaas.instances.get(instance_info.id)
assert_equal(instance.volume['size'], self.new_volume_size)
def wait_for_failure_status(self):
"""Wait until status becomes running."""
def is_finished_rebooting():
instance = self.instance
if instance.status == "REBOOT" or instance.status == "ACTIVE":
return False
# The reason we check for BLOCKED as well as SHUTDOWN is because
# Upstart might try to bring mysql back up after the borked
# connection and the guest status can be either
assert_true(instance.status in ("SHUTDOWN", "BLOCKED"))
return True
poll_until(is_finished_rebooting, time_out=TIME_OUT_TIME)
def test_instance_status_after_double_migrate(self):
"""
This test is to verify that instance status returned is more
informative than 'Status is {}'. There are several ways to
replicate this error. A double migration is just one of them but
since this is a known way to recreate that error we will use it
here to be sure that the error is fixed. The actual code lives
in reddwarf/instance/models.py in _validate_can_perform_action()
"""
# TODO(imsplitbit): test other instances where this issue could be
# replicated. Resizing a resized instance awaiting confirmation
# can be used as another case. This all boils back to the same
# piece of code so I'm not sure if it's relevant or not but could
# be done.
result = self.client.instances.create('testbox', 1, {'size': 5})
id = result.id
self.instances.append(id)
def verify_instance_is_active():
result = self.client.instances.get(id)
print result.status
return result.status == 'ACTIVE'
def attempt_migrate():
print 'attempting migration'
try:
self.mgmt.migrate(id)
except exceptions.UnprocessableEntity:
return False
return True
# Timing necessary to make the error occur
poll_until(verify_instance_is_active, time_out=120, sleep_time=1)
try:
poll_until(attempt_migrate, time_out=10, sleep_time=1)
except rd_exceptions.PollTimeOut:
fail('Initial migration timed out')
try:
self.mgmt.migrate(id)
except exceptions.UnprocessableEntity as err:
assert('status was {}' not in err.message)
else:
# If we are trying to test what status is returned when an
# instance is in a confirm_resize state and another
# migration is attempted then we also need to
# assert that an exception is raised when running migrate.
# If one is not then we aren't able to test what the
# returned status is in the exception message.
fail('UnprocessableEntity was not thrown')
def test_volume_resize_success(self):
def check_resize_status():
instance = instance_info.dbaas.instances.get(instance_info.id)
if instance.status == "ACTIVE":
return True
elif instance.status == "RESIZE":
return False
else:
fail("Status should not be %s" % instance.status)
poll_until(check_resize_status, sleep_time=2, time_out=300)
instance = instance_info.dbaas.instances.get(instance_info.id)
assert_equal(instance.volume['size'], self.new_volume_size)
def create_user(self):
"""Create a MySQL user we can use for this test."""
users = [{"name": MYSQL_USERNAME, "password": MYSQL_PASSWORD,
"database": MYSQL_USERNAME}]
self.dbaas.users.create(instance_info.id, users)
def has_user():
users = self.dbaas.users.list(instance_info.id)
return any([user.name == MYSQL_USERNAME for user in users])
poll_until(has_user, time_out=30)
if not FAKE_MODE:
time.sleep(5)
def test_verify_backup(self):
result = self._create_backup(BACKUP_NAME, BACKUP_DESC)
assert_equal(BACKUP_NAME, result.name)
assert_equal(BACKUP_DESC, result.description)
assert_equal(instance_info.id, result.instance_id)
assert_equal('NEW', result.status)
instance = instance_info.dbaas.instances.list()[0]
assert_equal('BACKUP', instance.status)
global backup_info
backup_info = result
# print dir(backup_info)
# print backup_info.locationRef
# Timing necessary to make the error occur
poll_until(self._verify_instance_is_active, time_out=120, sleep_time=1)
def setUp(self):
self.user = test_config.users.find_user(Requirements(is_admin=True))
self.client = create_dbaas_client(self.user)
self.name = 'test_SERVER_ERROR'
# Create an instance with a broken compute instance.
self.response = self.client.instances.create(self.name, 1, {'size': 1},
[])
poll_until(lambda: self.client.instances.get(self.response.id),
lambda instance: instance.status == 'ERROR',
time_out=10)
self.instance = self.client.instances.get(self.response.id)
print "Status: %s" % self.instance.status
msg = "Instance did not drop to error after server prov failure."
assert_equal(self.instance.status, "ERROR", msg)
def test_backup_delete(self):
"""test delete"""
instance_info.dbaas.backups.delete(backup_info.id)
assert_equal(202, instance_info.dbaas.last_http_code)
def backup_is_gone():
result = instance_info.dbaas.instances.backups(instance_info.id)
if len(result) == 0:
return True
else:
return False
poll_until(backup_is_gone)
assert_raises(exceptions.NotFound, instance_info.dbaas.backups.get,
backup_info.id)
def setUp(self):
self.user = test_config.users.find_user(Requirements(is_admin=True))
self.client = create_dbaas_client(self.user)
self.name = 'test_SERVER_ERROR'
# Create an instance with a broken compute instance.
self.response = self.client.instances.create(self.name, 1,
{'size': 1}, [])
poll_until(lambda: self.client.instances.get(self.response.id),
lambda instance: instance.status == 'ERROR',
time_out=10)
self.instance = self.client.instances.get(self.response.id)
print "Status: %s" % self.instance.status
msg = "Instance did not drop to error after server prov failure."
assert_equal(self.instance.status, "ERROR", msg)
def test_bad_resize_vol_data(self):
def _check_instance_status():
inst = self.dbaas.instances.get(self.instance)
if inst.status == "ACTIVE":
return True
else:
return False
poll_until(_check_instance_status)
try:
self.dbaas.instances.resize_volume(self.instance.id, "bad data")
except Exception as e:
resp, body = self.dbaas.client.last_response
httpCode = resp.status
assert_true(httpCode == 400, "Resize instance failed with code %s, exception %s" % (httpCode, e))
def test_backup_delete(self):
"""test delete"""
instance_info.dbaas.backups.delete(backup_info.id)
assert_equal(202, instance_info.dbaas.last_http_code)
def backup_is_gone():
result = instance_info.dbaas.instances.backups(instance_info.id)
if len(result) == 0:
return True
else:
return False
poll_until(backup_is_gone)
assert_raises(exceptions.NotFound, instance_info.dbaas.backups.get,
backup_info.id)
def test_verify_backup(self):
result = self._create_backup(BACKUP_NAME, BACKUP_DESC)
assert_equal(BACKUP_NAME, result.name)
assert_equal(BACKUP_DESC, result.description)
assert_equal(instance_info.id, result.instance_id)
assert_equal('NEW', result.status)
instance = instance_info.dbaas.instances.list()[0]
assert_equal('BACKUP', instance.status)
global backup_info
backup_info = result
# print dir(backup_info)
# print backup_info.locationRef
# Timing necessary to make the error occur
poll_until(self._verify_instance_is_active, time_out=120,
sleep_time=1)
def set_up(self):
"""Create client for mgmt instance test (2)."""
if not CONFIG.fake_mode:
raise SkipTest("This test only works in fake mode.")
self.client = create_client(is_admin=True)
self.mgmt = self.client.management
# Fake nova will fail a server ending with 'test_SERVER_ERROR'."
# Fake volume will fail if the size is 13.
# TODO(tim.simpson): This would be a lot nicer looking if we used a
# traditional mock framework.
response = self.client.instances.create("test_SERVER_ERROR", 1, {"size": 13}, [])
poll_until(
lambda: self.client.instances.get(response.id), lambda instance: instance.status == "ERROR", time_out=10
)
self.id = response.id
def set_up(self):
"""Create client for mgmt instance test (2)."""
if not CONFIG.fake_mode:
raise SkipTest("This test only works in fake mode.")
self.client = create_client(is_admin=True)
self.mgmt = self.client.management
# Fake nova will fail a server ending with 'test_SERVER_ERROR'."
# Fake volume will fail if the size is 13.
# TODO(tim.simpson): This would be a lot nicer looking if we used a
# traditional mock framework.
response = self.client.instances.create('test_SERVER_ERROR', 1,
{'size': 13}, [])
poll_until(lambda: self.client.instances.get(response.id),
lambda instance: instance.status == 'ERROR',
time_out=10)
self.id = response.id
def test_bad_revoke_user_access(self):
db = ""
def _check_instance_status():
inst = self.dbaas.instances.get(self.instance)
if inst.status == "ACTIVE":
return True
else:
return False
poll_until(_check_instance_status)
try:
self.dbaas.users.revoke(self.instance, self.user, db)
except Exception as e:
resp, body = self.dbaas.client.last_response
httpCode = resp.status
assert_true(httpCode == 404, "Revoke user access failed w/code %s, exception %s" % (httpCode, e))
def test_bad_change_user_password(self):
users = [{"name": ""}]
def _check_instance_status():
inst = self.dbaas.instances.get(self.instance)
if inst.status == "ACTIVE":
return True
else:
return False
poll_until(_check_instance_status)
try:
self.dbaas.users.change_passwords(self.instance, users)
except Exception as e:
resp, body = self.dbaas.client.last_response
httpCode = resp.status
assert_true(httpCode == 400, "Change usr/passwd failed with code %s, exception %s" % (httpCode, e))
def test_delete_restored_instance(self):
"""test delete restored instance"""
if test_config.auth_strategy == "fake":
raise SkipTest("Skipping delete restored instance for fake mode.")
instance_info.dbaas.instances.delete(restore_instance_id)
assert_equal(202, instance_info.dbaas.last_http_code)
def instance_is_gone():
try:
instance_info.dbaas.instances.get(restore_instance_id)
return False
except exceptions.NotFound:
return True
poll_until(instance_is_gone)
assert_raises(exceptions.NotFound, instance_info.dbaas.instances.get,
restore_instance_id)
def create_user(self):
"""Create a MySQL user we can use for this test."""
users = [{
"name": MYSQL_USERNAME,
"password": MYSQL_PASSWORD,
"database": MYSQL_USERNAME
}]
self.dbaas.users.create(instance_info.id, users)
def has_user():
users = self.dbaas.users.list(instance_info.id)
return any([user.name == MYSQL_USERNAME for user in users])
poll_until(has_user, time_out=30)
if not FAKE_MODE:
time.sleep(5)
def test_instance_restored(self):
if test_config.auth_strategy == "fake":
raise SkipTest("Skipping restore tests for fake mode.")
# This version just checks the REST API status.
def result_is_active():
instance = instance_info.dbaas.instances.get(restore_instance_id)
if instance.status == "ACTIVE":
return True
else:
# If its not ACTIVE, anything but BUILD must be
# an error.
assert_equal("BUILD", instance.status)
if instance_info.volume is not None:
assert_equal(instance.volume.get('used', None), None)
return False
poll_until(result_is_active)
def test_bad_resize_vol_data(self):
def _check_instance_status():
inst = self.dbaas.instances.get(self.instance)
if inst.status == "ACTIVE":
return True
else:
return False
poll_until(_check_instance_status)
try:
self.dbaas.instances.resize_volume(self.instance.id, "bad data")
except Exception as e:
resp, body = self.dbaas.client.last_response
httpCode = resp.status
assert_true(
httpCode == 400,
"Resize instance failed with code %s, exception %s" %
(httpCode, e))
def test_delete_backup(self):
print self.backup_id
self._delete_backup(self.backup_id)
assert_equal(202, instance_info.dbaas.last_http_code)
def backup_is_gone():
result = None
try:
result = instance_info.dbaas.backups.get(self.backup_id)
except exceptions.NotFound:
assert_equal(result.status, "404",
"Backup Resource not found)")
finally:
#print dir(result)
if result is None:
return True
else:
return False
poll_until(backup_is_gone)
def test_instance_restored(self):
if test_config.auth_strategy == "fake":
raise SkipTest("Skipping restore tests for fake mode.")
# This version just checks the REST API status.
def result_is_active():
instance = instance_info.dbaas.instances.get(restore_instance_id)
if instance.status == "ACTIVE":
return True
else:
# If its not ACTIVE, anything but BUILD must be
# an error.
assert_equal("BUILD", instance.status)
if instance_info.volume is not None:
assert_equal(instance.volume.get('used', None), None)
return False
poll_until(result_is_active)
def test_bad_change_user_password(self):
users = [{"name": ""}]
def _check_instance_status():
inst = self.dbaas.instances.get(self.instance)
if inst.status == "ACTIVE":
return True
else:
return False
poll_until(_check_instance_status)
try:
self.dbaas.users.change_passwords(self.instance, users)
except Exception as e:
resp, body = self.dbaas.client.last_response
httpCode = resp.status
assert_true(
httpCode == 400,
"Change usr/passwd failed with code %s, exception %s" %
(httpCode, e))
def test_bad_revoke_user_access(self):
db = ""
def _check_instance_status():
inst = self.dbaas.instances.get(self.instance)
if inst.status == "ACTIVE":
return True
else:
return False
poll_until(_check_instance_status)
try:
self.dbaas.users.revoke(self.instance, self.user, db)
except Exception as e:
resp, body = self.dbaas.client.last_response
httpCode = resp.status
assert_true(
httpCode == 404,
"Revoke user access failed w/code %s, exception %s" %
(httpCode, e))
def set_up(self):
"""Create client for mgmt instance test (2)."""
if not CONFIG.fake_mode:
raise SkipTest("This test only works in fake mode.")
self.client = create_client(is_admin=True)
self.mgmt = self.client.management
# Fake nova will fail a server ending with 'test_SERVER_ERROR'."
# Fake volume will fail if the size is 13.
# TODO(tim.simpson): This would be a lot nicer looking if we used a
# traditional mock framework.
body = None
if CONFIG.reddwarf_volume_support:
body = {'size': 13}
response = self.client.instances.create(
'test_SERVER_ERROR',
instance_info.dbaas_flavor_href,
body,
[])
poll_until(lambda: self.client.instances.get(response.id),
lambda instance: instance.status == 'ERROR',
time_out=10)
self.id = response.id
def test_instance_created(self):
# This version just checks the REST API status.
def result_is_active():
instance = dbaas.instances.get(instance_info.id)
if instance.status == "ACTIVE":
return True
else:
# If its not ACTIVE, anything but BUILD must be
# an error.
assert_equal("BUILD", instance.status)
if instance_info.volume is not None:
assert_equal(instance.volume.get('used', None), None)
return False
poll_until(result_is_active)
result = dbaas.instances.get(instance_info.id)
report = CONFIG.get_report()
report.log("Created an instance, ID = %s." % instance_info.id)
report.log("TIP:")
report.log("Rerun the tests with TESTS_USE_INSTANCE_ID=%s "
"to skip ahead to this point." % instance_info.id)
report.log("Add TESTS_DO_NOT_DELETE_INSTANCE=True to avoid deleting "
"the instance at the end of the tests.")
def test_verify_backup(self):
result = self._create_backup(BACKUP_NAME, BACKUP_DESC)
assert_equal(BACKUP_NAME, result.name)
assert_equal(BACKUP_DESC, result.description)
assert_equal(instance_info.id, result.instance_id)
assert_equal('NEW', result.status)
instance = instance_info.dbaas.instances.list()[0]
assert_equal('BACKUP', instance.status)
self.backup_id = result.id
# global backup_info
# backup_info = result
# print dir(backup_info)
# print dir(backup_info.id)
# print backup_info.id
# print backup_info.locationRef
# Timing necessary to make the error occur
#poll_until(self._verify_instance_is_active, time_out=120,
# sleep_time=1)
print "Polling starts"
poll_until(lambda: self._verify_backup_status(self.backup_id, "NEW"),
time_out=120, sleep_time=1)
print "After First Polling"
poll_until(lambda: self._verify_instance_status(instance_info.id,
"BACKUP"), time_out=120, sleep_time=1)
print "After second Polling"
# poll_until(lambda: self._verify_backup_status(self.backup_id,
# "BUILDING"), time_out=120, sleep_time=1)
print "After third Polling"
poll_until(lambda: self._verify_backup_status(self.backup_id,
"COMPLETED"), time_out=120, sleep_time=1)
print "After fourth Polling"
poll_until(lambda: self._verify_instance_status(instance_info.id,
"ACTIVE"), time_out=120, sleep_time=1)
# poll_until(self._verify_instance_is_active, time_out=120,
# sleep_time=1)
print "After fifth Polling"
def _wait_for_new_volume_size(self, new_size):
poll_until(lambda: self.client.instances.get(self.id),
lambda instance: instance.volume['size'] == new_size,
time_out=(60 * 8))
def wait_for_instance_task_status(self, instance_id, description):
poll_until(lambda: self.dbaas.management.show(instance_id),
lambda instance: instance.task_description == description,
time_out=10)
def wait_for_instance_status(self, instance_id, status="ACTIVE"):
poll_until(lambda: self.dbaas.instances.get(instance_id),
lambda instance: instance.status == status,
time_out=10)
def wait_for_instance_task_status(self, instance_id, description):
poll_until(lambda: self.dbaas.management.show(instance_id),
lambda instance: instance.task_description == description,
time_out=10)
def _wait_for_active(self):
poll_until(lambda: self.client.instances.get(self.id),
lambda instance: instance.status == "ACTIVE",
time_out=(60 * 8))
def resize_volume_in_shutdown_state(self):
self.client.instances.resize_volume(self.id, 2)
poll_until(lambda: self.client.instances.get(self.id),
lambda instance: instance.volume['size'] == 2,
time_out=(60 * 8))
def _wait_for_active(self):
poll_until(lambda: self.client.instances.get(self.id),
lambda instance: instance.status == "ACTIVE",
time_out=(60 * 8))
