def destroy_guest_and_wait_for_failure(self):
"""Make sure the Trove Compute Manager FAILS a timed-out guest."""
# Utterly kill the guest install.
process("sudo rm -rf /vz/private/%s/bin" % str(self.local_id))
# Make sure that before the timeout expires the guest state in the
# internal API and the REST API instance status is set to FAIL.
self.wait_for_rest_api_to_show_status_as_failed(time_out=GUEST_INSTALL_TIMEOUT + 30)
# At this point there is a tiny chance the compute API will spend a
# little bit of time trying to suspend the instance. We need it to
# because, while in this case we know the guest is dead, in the real
# world where this might happen for less-predictable reasons we want
# to make sure the misbehaving or just slow Nova-Guest daemon doesn't
# change its status to something besides FAILED before the instance is
# shut-off. So we have to make sure that the instance turns off, and
# the manager sets the guest state to FAILED afterwards.
self.wait_for_compute_instance_to_suspend()
# TODO(tim.simpson): It'd be really cool if we could somehow coax the
# guest to repeatedly setting its state in the db to
# something besides failed, so we could assert that
# no matter what after it was suspended it was set
# to such. Although maybe that's overkill.
self._assert_status_failure(self._get_status_tuple())
abort_count2 = count_notifications(notifier.ERROR, "trove.instance.abort.guest")
assert_true(self.abort_count < abort_count2)
def destroy_guest_and_wait_for_failure(self):
"""Make sure the Reddwarf Compute Manager FAILS a timed-out guest."""
# Utterly kill the guest install.
process("sudo rm -rf /vz/private/%s/bin" % str(self.local_id))
# Make sure that before the timeout expires the guest state in the
# internal API and the REST API instance status is set to FAIL.
self.wait_for_rest_api_to_show_status_as_failed(
time_out=GUEST_INSTALL_TIMEOUT + 30)
# At this point there is a tiny chance the compute API will spend a
# little bit of time trying to suspend the instance. We need it to
# because, while in this case we know the guest is dead, in the real
# world where this might happen for less-predictable reasons we want
# to make sure the misbehaving or just slow Nova-Guest daemon doesn't
# change its status to something besides FAILED before the instance is
# shut-off. So we have to make sure that the instance turns off, and
# the manager sets the guest state to FAILED afterwards.
self.wait_for_compute_instance_to_suspend()
#TODO(tim.simpson): It'd be really cool if we could somehow coax the
# guest to repeatedly setting its state in the db to
# something besides failed, so we could assert that
# no matter what after it was suspended it was set
# to such. Although maybe that's overkill.
self._assert_status_failure(self._get_status_tuple())
abort_count2 = count_notifications(notifier.ERROR,
"reddwarf.instance.abort.guest")
assert_true(self.abort_count < abort_count2)
def mess_up_mysql(self):
"""Ruin MySQL's ability to restart."""
self.fix_mysql() # kill files
cmd = """ssh %s 'sudo cp /dev/null /var/lib/mysql/ib_logfile%d'"""
for index in range(2):
full_cmd = cmd % (instance_info.get_address(), index)
print("RUNNING COMMAND: %s" % full_cmd)
util.process(full_cmd)
def mess_up_mysql(self):
"""Ruin MySQL's ability to restart."""
self.fix_mysql() # kill files
cmd = """ssh %s 'sudo cp /dev/null /var/lib/mysql/ib_logfile%d'"""
for index in range(2):
full_cmd = cmd % (instance_info.get_address(), index)
print("RUNNING COMMAND: %s" % full_cmd)
util.process(full_cmd)
def test_30_check_options(self):
cmd = "sudo dumpe2fs -h %s 2> /dev/null | " \
"awk -F ':' '{ if($1 == \"Reserved block count\") { rescnt=$2 } }" \
" { if($1 == \"Block count\") { blkcnt=$2 } } END " \
"{ print (rescnt/blkcnt)*100 }'" % self.story.device_path
out, err = util.process(cmd)
self.assertEqual(float(5), round(float(out)), msg=out)
def test_mount_options(self):
cmd = "mount -l | awk '/%s.*noatime/ { print $1 }'"
cmd %= LOCAL_MOUNT_PATH.replace('/', '')
out, err = util.process(cmd)
self.assertEqual(os.path.realpath(self.story.device_path),
out.strip(),
msg=out)
def check_status_of_instance():
status, err = process("sudo vzctl status %s | awk '{print $5}'" % str(instance_info.local_id))
if string_in_list(status, ["running"]):
self.assertEqual("running", status.strip())
return True
else:
return False
def test_30_check_options(self):
cmd = ("sudo dumpe2fs -h %s 2> /dev/null | "
"awk -F ':' '{ if($1 == \"Reserved block count\") "
"{ rescnt=$2 } } { if($1 == \"Block count\") "
"{ blkcnt=$2 } } END { print (rescnt/blkcnt)*100 }'")
cmd = cmd % self.story.device_path
out, err = util.process(cmd)
self.assertEqual(float(5), round(float(out)), msg=out)
def show_databases(self, user, password):
cmd = "sudo mysql -h %s -u '%s' -p'%s' -e 'show databases;'" % (instance_info.user_ip, user, password)
print("Running cmd: %s" % cmd)
dblist, err = process(cmd)
print("returned: %s" % dblist)
if err:
assert_false(True, err)
return dblist
def check_status_of_instance():
status, err = process("sudo vzctl status %s | awk '{print $5}'" %
str(instance_info.local_id))
if string_in_list(status, ["running"]):
self.assertEqual("running", status.strip())
return True
else:
return False
def show_databases(self, user, password):
cmd = "sudo mysql -h %s -u '%s' -p'%s' -e 'show databases;'"\
% (instance_info.get_address(), user, password)
print("Running cmd: %s" % cmd)
dblist, err = process(cmd)
print("returned: %s" % dblist)
if err:
assert_false(True, err)
return dblist
def wait_for_compute_instance_to_start(self):
"""Wait for the compute instance to begin."""
while True:
status, err = process("sudo vzctl status %s | awk '{print $5}'" % str(self.local_id))
if not string_in_list(status, ["running"]):
time.sleep(5)
else:
assert_equal("running", status.strip())
break
def wait_for_compute_instance_to_start(self):
"""Wait for the compute instance to begin."""
while True:
status, err = process("sudo vzctl status %s | awk '{print $5}'"
% str(self.local_id))
if not string_in_list(status, ["running"]):
time.sleep(5)
else:
assert_equal("running", status.strip())
break
def wait_for_pid(self):
"""Wait for instance PID."""
pid = None
while pid is None:
guest_status = dbapi.guest_status_get(self.local_id)
rest_api_result = self.dbaas.instances.get(self.id)
cmd = "pstree -ap | grep init | cut -d',' -f2 | vzpid - | " "grep %s | awk '{print $1}'"
out, err = process(cmd % str(self.local_id))
pid = out.strip()
if not pid:
# Make sure the guest status is BUILDING during this time.
assert_equal(guest_status.state, power_state.BUILDING)
# REST API should return BUILDING as the status as well.
assert_equal(dbaas_mapping[power_state.BUILDING], rest_api_result.status)
time.sleep(10)
def wait_for_pid(self):
"""Wait for instance PID."""
pid = None
while pid is None:
guest_status = dbapi.guest_status_get(self.local_id)
rest_api_result = self.dbaas.instances.get(self.id)
cmd = ("pstree -ap | grep init | cut -d',' -f2 | vzpid - | "
"grep %s | awk '{print $1}'")
out, err = process(cmd % str(self.local_id))
pid = out.strip()
if not pid:
# Make sure the guest status is BUILDING during this time.
assert_equal(guest_status.state, power_state.BUILDING)
# REST API should return BUILDING as the status as well.
assert_equal(dbaas_mapping[power_state.BUILDING],
rest_api_result.status)
time.sleep(10)
def test_guest_process(self):
init_proc = re.compile("[\w\W\|\-\s\d,]*nova-guest --flagfile=/etc/nova/nova.conf nova[\W\w\s]*")
guest_proc = re.compile("[\w\W\|\-\s]*/usr/bin/nova-guest --flagfile=/etc/nova/nova.conf[\W\w\s]*")
apt = re.compile("[\w\W\|\-\s]*apt-get[\w\W\|\-\s]*")
while True:
guest_process, err = process("pstree -ap %s | grep nova-guest"
% instance_info.pid)
if not string_in_list(guest_process, ["nova-guest"]):
time.sleep(10)
else:
if apt.match(guest_process):
time.sleep(10)
else:
init = init_proc.match(guest_process)
guest = guest_proc.match(guest_process)
if init and guest:
self.assertTrue(True, init.group())
else:
self.assertFalse(False, guest_process)
break
def check_process_alive_via_local_ovz(self):
init_re = "[\w\W\|\-\s\d,]*nova-guest " "--flagfile=/etc/nova/nova.conf nova[\W\w\s]*"
init_proc = re.compile(init_re)
guest_re = "[\w\W\|\-\s]*/usr/bin/nova-guest " "--flagfile=/etc/nova/nova.conf[\W\w\s]*"
guest_proc = re.compile(guest_re)
apt = re.compile("[\w\W\|\-\s]*apt-get[\w\W\|\-\s]*")
while True:
guest_process, err = process("pstree -ap %s | grep nova-guest" % instance_info.pid)
if not string_in_list(guest_process, ["nova-guest"]):
time.sleep(10)
else:
if apt.match(guest_process):
time.sleep(10)
else:
init = init_proc.match(guest_process)
guest = guest_proc.match(guest_process)
if init and guest:
assert_true(True, init.group())
else:
assert_false(False, guest_process)
break
def check_process_alive_via_local_ovz(self):
init_re = ("[\w\W\|\-\s\d,]*nova-guest "
"--flagfile=/etc/nova/nova.conf nova[\W\w\s]*")
init_proc = re.compile(init_re)
guest_re = ("[\w\W\|\-\s]*/usr/bin/nova-guest "
"--flagfile=/etc/nova/nova.conf[\W\w\s]*")
guest_proc = re.compile(guest_re)
apt = re.compile("[\w\W\|\-\s]*apt-get[\w\W\|\-\s]*")
while True:
guest_process, err = process("pstree -ap %s | grep nova-guest" %
instance_info.pid)
if not string_in_list(guest_process, ["nova-guest"]):
time.sleep(10)
else:
if apt.match(guest_process):
time.sleep(10)
else:
init = init_proc.match(guest_process)
guest = guest_proc.match(guest_process)
if init and guest:
assert_true(True, init.group())
else:
assert_false(False, guest_process)
break
def fix_mysql(self):
"""Fix MySQL's ability to restart."""
if not FAKE_MODE:
cmd = "ssh %s 'sudo rm /var/lib/mysql/ib_logfile%d'"
for index in range(2):
util.process(cmd % (instance_info.get_address(), index))
def fix_mysql(self):
"""Fix MySQL's ability to restart."""
if not FAKE_MODE:
cmd = "ssh %s 'sudo rm /var/lib/mysql/ib_logfile%d'"
for index in range(2):
util.process(cmd % (instance_info.get_address(), index))
def fix_mysql(self):
"""Fix MySQL's ability to restart."""
cmd = "sudo vzctl exec %d rm /var/lib/mysql/ib_logfile%d"
for index in range(2):
util.process(cmd % (self.instance_local_id, index))
def test_mount_options(self):
cmd = "mount -l | awk '/%s.*noatime/ { print $1 }'"
cmd %= LOCAL_MOUNT_PATH.replace('/','')
out, err = util.process(cmd)
self.assertEqual(os.path.realpath(self.story.device_path), out.strip(),
msg=out)
def get_the_pid():
out, err = process("pgrep init | vzpid - | awk '/%s/{print $1}'" %
str(instance_info.local_id))
instance_info.pid = out.strip()
return len(instance_info.pid) > 0
def get_the_pid():
out, err = process("pgrep init | vzpid - | awk '/%s/{print $1}'"
% str(instance_info.local_id))
instance_info.pid = out.strip()
return len(instance_info.pid) > 0
def mess_up_mysql(self):
"""Ruin MySQL's ability to restart."""
self.fix_mysql() # kill files
cmd = """sudo vzctl exec %d 'echo "hi" > /var/lib/mysql/ib_logfile%d'"""
for index in range(2):
util.process(cmd % (self.instance_local_id, index))