def destroy_guest_and_wait_for_failure(self):
        """Make sure the Trove Compute Manager FAILS a timed-out guest."""

        # Utterly kill the guest install.
        process("sudo rm -rf /vz/private/%s/bin" % str(self.local_id))

        # Make sure that before the timeout expires the guest state in the
        # internal API and the REST API instance status is set to FAIL.
        self.wait_for_rest_api_to_show_status_as_failed(time_out=GUEST_INSTALL_TIMEOUT + 30)

        # At this point there is a tiny chance the compute API will spend a
        # little bit of time trying to suspend the instance. We need it to
        # because, while in this case we know the guest is dead, in the real
        # world where this might happen for less-predictable reasons we want
        # to make sure the misbehaving or just slow Nova-Guest daemon doesn't
        # change its status to something besides FAILED before the instance is
        # shut-off. So we have to make sure that the instance turns off, and
        # the manager sets the guest state to FAILED afterwards.
        self.wait_for_compute_instance_to_suspend()

        # TODO(tim.simpson): It'd be really cool if we could somehow coax the
        #                   guest to repeatedly setting its state in the db to
        #                   something besides failed, so we could assert that
        #                   no matter what after it was suspended it was set
        #                   to such.  Although maybe that's overkill.
        self._assert_status_failure(self._get_status_tuple())
        abort_count2 = count_notifications(notifier.ERROR, "trove.instance.abort.guest")
        assert_true(self.abort_count < abort_count2)
Esempio n. 2
0
    def destroy_guest_and_wait_for_failure(self):
        """Make sure the Reddwarf Compute Manager FAILS a timed-out guest."""

        # Utterly kill the guest install.
        process("sudo rm -rf /vz/private/%s/bin" % str(self.local_id))

        # Make sure that before the timeout expires the guest state in the
        # internal API and the REST API instance status is set to FAIL.
        self.wait_for_rest_api_to_show_status_as_failed(
            time_out=GUEST_INSTALL_TIMEOUT + 30)

        # At this point there is a tiny chance the compute API will spend a
        # little bit of time trying to suspend the instance. We need it to
        # because, while in this case we know the guest is dead, in the real
        # world where this might happen for less-predictable reasons we want
        # to make sure the misbehaving or just slow Nova-Guest daemon doesn't
        # change its status to something besides FAILED before the instance is
        # shut-off. So we have to make sure that the instance turns off, and
        # the manager sets the guest state to FAILED afterwards.
        self.wait_for_compute_instance_to_suspend()

        #TODO(tim.simpson): It'd be really cool if we could somehow coax the
        #                   guest to repeatedly setting its state in the db to
        #                   something besides failed, so we could assert that
        #                   no matter what after it was suspended it was set
        #                   to such.  Although maybe that's overkill.
        self._assert_status_failure(self._get_status_tuple())
        abort_count2 = count_notifications(notifier.ERROR,
                                           "reddwarf.instance.abort.guest")
        assert_true(self.abort_count < abort_count2)
 def mess_up_mysql(self):
     """Ruin MySQL's ability to restart."""
     self.fix_mysql()  # kill files
     cmd = """ssh %s 'sudo cp /dev/null /var/lib/mysql/ib_logfile%d'"""
     for index in range(2):
         full_cmd = cmd % (instance_info.get_address(), index)
         print("RUNNING COMMAND: %s" % full_cmd)
         util.process(full_cmd)
Esempio n. 4
0
 def mess_up_mysql(self):
     """Ruin MySQL's ability to restart."""
     self.fix_mysql()  # kill files
     cmd = """ssh %s 'sudo cp /dev/null /var/lib/mysql/ib_logfile%d'"""
     for index in range(2):
         full_cmd = cmd % (instance_info.get_address(), index)
         print("RUNNING COMMAND: %s" % full_cmd)
         util.process(full_cmd)
 def test_30_check_options(self):
     cmd = "sudo dumpe2fs -h %s 2> /dev/null | " \
           "awk -F ':' '{ if($1 == \"Reserved block count\") { rescnt=$2 } }" \
           " { if($1 == \"Block count\") { blkcnt=$2 } } END " \
           "{ print (rescnt/blkcnt)*100 }'" % self.story.device_path
     out, err = util.process(cmd)
     self.assertEqual(float(5), round(float(out)), msg=out)
Esempio n. 6
0
 def test_mount_options(self):
     cmd = "mount -l | awk '/%s.*noatime/ { print $1 }'"
     cmd %= LOCAL_MOUNT_PATH.replace('/', '')
     out, err = util.process(cmd)
     self.assertEqual(os.path.realpath(self.story.device_path),
                      out.strip(),
                      msg=out)
 def check_status_of_instance():
     status, err = process("sudo vzctl status %s | awk '{print $5}'" % str(instance_info.local_id))
     if string_in_list(status, ["running"]):
         self.assertEqual("running", status.strip())
         return True
     else:
         return False
Esempio n. 8
0
 def test_30_check_options(self):
     cmd = ("sudo dumpe2fs -h %s 2> /dev/null | "
            "awk -F ':' '{ if($1 == \"Reserved block count\") "
            "{ rescnt=$2 } } { if($1 == \"Block count\") "
            "{ blkcnt=$2 } } END { print (rescnt/blkcnt)*100 }'")
     cmd = cmd % self.story.device_path
     out, err = util.process(cmd)
     self.assertEqual(float(5), round(float(out)), msg=out)
 def show_databases(self, user, password):
     cmd = "sudo mysql -h %s -u '%s' -p'%s' -e 'show databases;'" % (instance_info.user_ip, user, password)
     print("Running cmd: %s" % cmd)
     dblist, err = process(cmd)
     print("returned: %s" % dblist)
     if err:
         assert_false(True, err)
     return dblist
Esempio n. 10
0
 def check_status_of_instance():
     status, err = process("sudo vzctl status %s | awk '{print $5}'" %
                           str(instance_info.local_id))
     if string_in_list(status, ["running"]):
         self.assertEqual("running", status.strip())
         return True
     else:
         return False
Esempio n. 11
0
 def show_databases(self, user, password):
     cmd = "sudo mysql -h %s -u '%s' -p'%s' -e 'show databases;'"\
             % (instance_info.get_address(), user, password)
     print("Running cmd: %s" % cmd)
     dblist, err = process(cmd)
     print("returned: %s" % dblist)
     if err:
         assert_false(True, err)
     return dblist
    def wait_for_compute_instance_to_start(self):
        """Wait for the compute instance to begin."""
        while True:
            status, err = process("sudo vzctl status %s | awk '{print $5}'" % str(self.local_id))

            if not string_in_list(status, ["running"]):
                time.sleep(5)
            else:
                assert_equal("running", status.strip())
                break
Esempio n. 13
0
    def wait_for_compute_instance_to_start(self):
        """Wait for the compute instance to begin."""
        while True:
            status, err = process("sudo vzctl status %s | awk '{print $5}'"
                                  % str(self.local_id))

            if not string_in_list(status, ["running"]):
                time.sleep(5)
            else:
                assert_equal("running", status.strip())
                break
 def wait_for_pid(self):
     """Wait for instance PID."""
     pid = None
     while pid is None:
         guest_status = dbapi.guest_status_get(self.local_id)
         rest_api_result = self.dbaas.instances.get(self.id)
         cmd = "pstree -ap | grep init | cut -d',' -f2 | vzpid - | " "grep %s | awk '{print $1}'"
         out, err = process(cmd % str(self.local_id))
         pid = out.strip()
         if not pid:
             # Make sure the guest status is BUILDING during this time.
             assert_equal(guest_status.state, power_state.BUILDING)
             # REST API should return BUILDING as the status as well.
             assert_equal(dbaas_mapping[power_state.BUILDING], rest_api_result.status)
             time.sleep(10)
Esempio n. 15
0
 def wait_for_pid(self):
     """Wait for instance PID."""
     pid = None
     while pid is None:
         guest_status = dbapi.guest_status_get(self.local_id)
         rest_api_result = self.dbaas.instances.get(self.id)
         cmd = ("pstree -ap | grep init | cut -d',' -f2 | vzpid - | "
                "grep %s | awk '{print $1}'")
         out, err = process(cmd % str(self.local_id))
         pid = out.strip()
         if not pid:
             # Make sure the guest status is BUILDING during this time.
             assert_equal(guest_status.state, power_state.BUILDING)
             # REST API should return BUILDING as the status as well.
             assert_equal(dbaas_mapping[power_state.BUILDING],
                          rest_api_result.status)
             time.sleep(10)
 def test_guest_process(self):
     init_proc = re.compile("[\w\W\|\-\s\d,]*nova-guest --flagfile=/etc/nova/nova.conf nova[\W\w\s]*")
     guest_proc = re.compile("[\w\W\|\-\s]*/usr/bin/nova-guest --flagfile=/etc/nova/nova.conf[\W\w\s]*")
     apt = re.compile("[\w\W\|\-\s]*apt-get[\w\W\|\-\s]*")
     while True:
         guest_process, err = process("pstree -ap %s | grep nova-guest"
                                         % instance_info.pid)
         if not string_in_list(guest_process, ["nova-guest"]):
             time.sleep(10)
         else:
             if apt.match(guest_process):
                 time.sleep(10)
             else:
                 init = init_proc.match(guest_process)
                 guest = guest_proc.match(guest_process)
                 if init and guest:
                     self.assertTrue(True, init.group())
                 else:
                     self.assertFalse(False, guest_process)
                 break
 def check_process_alive_via_local_ovz(self):
     init_re = "[\w\W\|\-\s\d,]*nova-guest " "--flagfile=/etc/nova/nova.conf nova[\W\w\s]*"
     init_proc = re.compile(init_re)
     guest_re = "[\w\W\|\-\s]*/usr/bin/nova-guest " "--flagfile=/etc/nova/nova.conf[\W\w\s]*"
     guest_proc = re.compile(guest_re)
     apt = re.compile("[\w\W\|\-\s]*apt-get[\w\W\|\-\s]*")
     while True:
         guest_process, err = process("pstree -ap %s | grep nova-guest" % instance_info.pid)
         if not string_in_list(guest_process, ["nova-guest"]):
             time.sleep(10)
         else:
             if apt.match(guest_process):
                 time.sleep(10)
             else:
                 init = init_proc.match(guest_process)
                 guest = guest_proc.match(guest_process)
                 if init and guest:
                     assert_true(True, init.group())
                 else:
                     assert_false(False, guest_process)
                 break
Esempio n. 18
0
 def check_process_alive_via_local_ovz(self):
     init_re = ("[\w\W\|\-\s\d,]*nova-guest "
                "--flagfile=/etc/nova/nova.conf nova[\W\w\s]*")
     init_proc = re.compile(init_re)
     guest_re = ("[\w\W\|\-\s]*/usr/bin/nova-guest "
                 "--flagfile=/etc/nova/nova.conf[\W\w\s]*")
     guest_proc = re.compile(guest_re)
     apt = re.compile("[\w\W\|\-\s]*apt-get[\w\W\|\-\s]*")
     while True:
         guest_process, err = process("pstree -ap %s | grep nova-guest" %
                                      instance_info.pid)
         if not string_in_list(guest_process, ["nova-guest"]):
             time.sleep(10)
         else:
             if apt.match(guest_process):
                 time.sleep(10)
             else:
                 init = init_proc.match(guest_process)
                 guest = guest_proc.match(guest_process)
                 if init and guest:
                     assert_true(True, init.group())
                 else:
                     assert_false(False, guest_process)
                 break
Esempio n. 19
0
 def fix_mysql(self):
     """Fix MySQL's ability to restart."""
     if not FAKE_MODE:
         cmd = "ssh %s 'sudo rm /var/lib/mysql/ib_logfile%d'"
         for index in range(2):
             util.process(cmd % (instance_info.get_address(), index))
 def fix_mysql(self):
     """Fix MySQL's ability to restart."""
     if not FAKE_MODE:
         cmd = "ssh %s 'sudo rm /var/lib/mysql/ib_logfile%d'"
         for index in range(2):
             util.process(cmd % (instance_info.get_address(), index))
 def fix_mysql(self):
     """Fix MySQL's ability to restart."""
     cmd = "sudo vzctl exec %d rm /var/lib/mysql/ib_logfile%d"
     for index in range(2):
         util.process(cmd % (self.instance_local_id, index))
 def test_mount_options(self):
     cmd = "mount -l | awk '/%s.*noatime/ { print $1 }'"
     cmd %= LOCAL_MOUNT_PATH.replace('/','')
     out, err = util.process(cmd)
     self.assertEqual(os.path.realpath(self.story.device_path), out.strip(),
                      msg=out)
Esempio n. 23
0
 def get_the_pid():
     out, err = process("pgrep init | vzpid - | awk '/%s/{print $1}'" %
                        str(instance_info.local_id))
     instance_info.pid = out.strip()
     return len(instance_info.pid) > 0
 def get_the_pid():
     out, err = process("pgrep init | vzpid - | awk '/%s/{print $1}'"
                         % str(instance_info.local_id))
     instance_info.pid = out.strip()
     return len(instance_info.pid) > 0
 def mess_up_mysql(self):
     """Ruin MySQL's ability to restart."""
     self.fix_mysql() # kill files
     cmd = """sudo vzctl exec %d 'echo "hi" > /var/lib/mysql/ib_logfile%d'"""
     for index in range(2):
         util.process(cmd % (self.instance_local_id, index))