Example #1
0
    def setUp(self):
        self.image_manager = MagicMock()
        self.vm_manager = MagicMock()
        self.image_scanner = DatastoreImageScanner(self.image_manager,
                                                   self.vm_manager,
                                                   self.DATASTORE_ID)
        self.synchronizer = TestSynchronizer()
        self.wait_at_the_end_of_scan = False

        self.raise_exception = False
        self.timeout = self.TIMEOUT
    def setUp(self):
        self.image_manager = MagicMock()
        self.vm_manager = MagicMock()
        self.image_scanner = DatastoreImageScanner(
            self.image_manager,
            self.vm_manager,
            self.DATASTORE_ID)
        self.synchronizer = TestSynchronizer()
        self.wait_at_the_end_of_scan = False

        self.raise_exception = False
        self.timeout = self.TIMEOUT
    def setUp(self):
        # Create VM manager
        self.vim_client = VimClient(auto_sync=False)
        self.vim_client._content = MagicMock()
        self.vim_client.wait_for_task = MagicMock()
        self.vm_manager = VmManager(self.vim_client, MagicMock())
        services.register(ServiceName.AGENT_CONFIG, MagicMock())

        # Set up test files
        self.base_dir = os.path.dirname(__file__)
        self.test_dir = os.path.join(self.base_dir, "../test_files")
        self.image_manager = ImageManager(MagicMock(), MagicMock())
        self.image_scanner = DatastoreImageScanner(self.image_manager, self.vm_manager, self.DATASTORE_ID)
        self.write_count = 0
 def __init__(self, datastore_manager, image_manager, vm_manager):
     self.logger = logging.getLogger(__name__)
     self.datastore_manager = datastore_manager
     self.datastore_image_scanners = dict()
     self.datastore_image_sweepers = dict()
     for datastore_id in self.datastore_manager.get_datastore_ids():
         self.logger.info("IMAGE SCANNER: adding datastore: %s" %
                          datastore_id)
         self.datastore_image_scanners[datastore_id] = \
             DatastoreImageScanner(image_manager,
                                   vm_manager,
                                   datastore_id)
         self.datastore_image_sweepers[datastore_id] = \
             DatastoreImageSweeper(image_manager,
                                   datastore_id)
    def setUp(self):
        self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
        services.register(ServiceName.AGENT_CONFIG, MagicMock())
        self.image_manager = ImageManager(MagicMock(), MagicMock())
        self.vm_manager = MagicMock()
        self.image_scanner = DatastoreImageScanner(self.image_manager,
                                                   self.vm_manager,
                                                   self.DATASTORE_ID)
        self.write_count = 0

        # Create various image directories and empty vmdks
        image_id_1 = str(uuid.uuid4())
        image_id_2 = str(uuid.uuid4())
        image_id_3 = str(uuid.uuid4())
        image_id_4 = "invalid_image_id"
        self.image_ids = ["*", image_id_1, image_id_2, image_id_3, image_id_4]
        dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
        os.makedirs(dir1)
        dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
        os.makedirs(dir2)
        dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
        os.makedirs(dir3)
        dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
        os.makedirs(dir4)
        # Create a vmdk under "im", since the image_id is not a valid uuid it should be skipped
        open(os.path.join(self.test_dir, "image_im.vmdk"), 'w').close()
        # Create a good image vmdk under image_id_1, the name of the vmdk matches the directory
        # that contains it so this is a valid image to remove
        vmdk_filename = image_id_1 + ".vmdk"
        open(os.path.join(dir1, vmdk_filename), 'w').close()
        # Create a good image vmdk under image_id_2, also create an unused image marker file,
        # image_id_2 should also be included in the list of images to remove
        vmdk_filename = image_id_2 + ".vmdk"
        open(os.path.join(dir2, vmdk_filename), 'w').close()
        open(
            os.path.join(dir2,
                         self.image_manager.UNUSED_IMAGE_MARKER_FILE_NAME),
            'w').close()
        # Don't create anything under directory dir3. it should still mark the image as deletable

        # Create a vmdk under an invalid image directory, since the image id is not valid it
        # should not mark it for deletion
        vmdk_filename = image_id_4 + ".vmdk"
        open(os.path.join(dir4, vmdk_filename), 'w').close()
Example #6
0
class ImageScannerTestCase(unittest.TestCase):
    DATASTORE_ID = "DS01"
    VM_SCAN_RATE = 11
    IMAGE_MARK_RATE = 12
    TIMEOUT = 6000

    # This test uses two threads, the main thread is used to check
    # the state inside the state machine at different stages. The
    # second thread, ImageScannerThread, executes the steps in
    # sequence and calls out the two main methods:
    # get_active_images() and mark_unused()
    # These methods have been patched to synchronize with the main
    # thread.
    def setUp(self):
        self.image_manager = MagicMock()
        self.vm_manager = MagicMock()
        self.image_scanner = DatastoreImageScanner(self.image_manager,
                                                   self.vm_manager,
                                                   self.DATASTORE_ID)
        self.synchronizer = TestSynchronizer()
        self.wait_at_the_end_of_scan = False

        self.raise_exception = False
        self.timeout = self.TIMEOUT

    def tearDown(self):
        self.image_scanner.stop()

    @patch(
        "host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_for_unused_images"
    )
    @patch(
        "host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_vms_for_active_images"
    )
    def test_lifecycle(self, scan_vms_for_active_images,
                       scan_for_unused_images):
        assert_that(
            self.image_scanner.get_state() is DatastoreImageScanner.State.IDLE)

        scan_vms_for_active_images.side_effect = self.fake_scan_vms_for_active_images
        scan_for_unused_images.side_effect = self.fake_scan_for_unused_images

        self.image_scanner.start(self.timeout, self.VM_SCAN_RATE,
                                 self.IMAGE_MARK_RATE)

        self.image_scanner.wait_for_task_end()

        assert_that(
            self.image_scanner.get_state() is DatastoreImageScanner.State.IDLE)

        scan_vms_for_active_images.assert_called_once_with(
            self.image_scanner, self.DATASTORE_ID)
        scan_for_unused_images.assert_called_once_with(self.image_scanner,
                                                       self.DATASTORE_ID)

    @patch(
        "host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_for_unused_images"
    )
    @patch(
        "host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_vms_for_active_images"
    )
    def test_stop(self, scan_vms_for_active_images, scan_for_unused_images):
        assert_that(
            self.image_scanner.get_state() is DatastoreImageScanner.State.IDLE)

        scan_vms_for_active_images.side_effect = self.fake_scan_vms_for_active_images
        scan_for_unused_images.side_effect = self.fake_scan_for_unused_images

        self.wait_at_the_end_of_scan = True
        self.image_scanner.start(self.timeout, self.VM_SCAN_RATE,
                                 self.IMAGE_MARK_RATE)

        self.wait_for_runner()
        self.image_scanner.stop()
        self.signal_runner()

        self.image_scanner.wait_for_task_end()

        assert_that(
            self.image_scanner.get_state() is DatastoreImageScanner.State.IDLE)
        exception = self.image_scanner.get_exception()
        assert_that(isinstance(exception, TaskTerminated) is True)

        scan_vms_for_active_images.assert_called_with(self.image_scanner,
                                                      self.DATASTORE_ID)
        assert_that(scan_for_unused_images.called is False)

    @patch(
        "host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_for_unused_images"
    )
    @patch(
        "host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_vms_for_active_images"
    )
    def test_timeout(self, scan_vms_for_active_images, scan_for_unused_images):
        assert_that(
            self.image_scanner.get_state() is DatastoreImageScanner.State.IDLE)

        scan_vms_for_active_images.side_effect = self.fake_scan_vms_for_active_images
        scan_for_unused_images.side_effect = self.fake_scan_for_unused_images

        self.timeout = 1
        self.wait_at_the_end_of_scan = True
        self.image_scanner.start(self.timeout, self.VM_SCAN_RATE,
                                 self.IMAGE_MARK_RATE)

        self.wait_for_runner()
        time.sleep(2)
        self.signal_runner()

        self.image_scanner.wait_for_task_end()

        assert_that(
            self.image_scanner.get_state() is DatastoreImageScanner.State.IDLE)
        exception = self.image_scanner.get_exception()
        assert_that(isinstance(exception, TaskTimeout) is True)

        scan_vms_for_active_images.assert_called_with(self.image_scanner,
                                                      self.DATASTORE_ID)
        assert_that(scan_for_unused_images.called is False)

    def test_parse_vmdk(self):
        base_dir = os.path.dirname(__file__)
        test_dir = os.path.join(base_dir, "../test_files", "vm_good")
        vmdk_pathname = os.path.join(test_dir, 'good.vmdk')
        dictionary = DatastoreImageScannerTaskRunner._parse_vmdk(vmdk_pathname)
        assert_that(dictionary["version"] == "1")
        assert_that(dictionary["encoding"] == "UTF-8")
        assert_that(dictionary["CID"] == "fffffffe")
        assert_that(dictionary["parentCID"] == "fffffffe")
        assert_that(dictionary["isNativeSnapshot"] == "no")
        assert_that(dictionary["createType"] == "vmfsSparse")
        assert_that(dictionary["parentFileNameHint"] ==
                    "/vmfs/volumes/555ca9f8-9f24fa2c-41c1-0025b5414043/"
                    "image_92e62599-6689-4a8f-ba2a-633914b5048e/92e"
                    "62599-6689-4a8f-ba2a-633914b5048e.vmdk")

    def fake_scan_vms_for_active_images(self, image_scanner, datastore):
        assert_that(image_scanner.datastore_id is self.DATASTORE_ID)
        assert_that(datastore is self.DATASTORE_ID)
        assert_that(self.image_scanner.get_state() is
                    DatastoreImageScanner.State.VM_SCAN)
        assert_that(image_scanner.vm_scan_rate is self.VM_SCAN_RATE)
        if self.wait_at_the_end_of_scan:
            # Wait here until the main thread wants to continue
            self.signal_main()
            self.wait_for_main()
        return dict()

    def fake_scan_for_unused_images(self, image_scanner, datastore):
        assert_that(image_scanner.datastore_id is self.DATASTORE_ID)
        assert_that(datastore is self.DATASTORE_ID)
        assert_that(self.image_scanner.get_state() is
                    DatastoreImageScanner.State.IMAGE_MARK)
        assert_that(image_scanner.image_mark_rate is self.IMAGE_MARK_RATE)
        return dict()

    def wait_for_runner(self):
        self.synchronizer.wait_for_runner()

    def signal_runner(self):
        self.synchronizer.signal_runner()

    def wait_for_main(self):
        self.synchronizer.wait_for_main()

    def signal_main(self):
        self.synchronizer.signal_main()
class ImageScannerTestCase(unittest.TestCase):
    DATASTORE_ID = "DS01"
    VM_SCAN_RATE = 11
    IMAGE_MARK_RATE = 12
    TIMEOUT = 6000

    # This test uses two threads, the main thread is used to check
    # the state inside the state machine at different stages. The
    # second thread, ImageScannerThread, executes the steps in
    # sequence and calls out the two main methods:
    # get_active_images() and mark_unused()
    # These methods have been patched to synchronize with the main
    # thread.
    def setUp(self):
        self.image_manager = MagicMock()
        self.vm_manager = MagicMock()
        self.image_scanner = DatastoreImageScanner(
            self.image_manager,
            self.vm_manager,
            self.DATASTORE_ID)
        self.synchronizer = TestSynchronizer()
        self.wait_at_the_end_of_scan = False

        self.raise_exception = False
        self.timeout = self.TIMEOUT

    def tearDown(self):
        self.image_scanner.stop()

    @patch("host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_for_unused_images")
    @patch("host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_vms_for_active_images")
    def test_lifecycle(self, scan_vms_for_active_images, scan_for_unused_images):
        assert_that(self.image_scanner.get_state() is
                    DatastoreImageScanner.State.IDLE)

        scan_vms_for_active_images.side_effect = self.fake_scan_vms_for_active_images
        scan_for_unused_images.side_effect = self.fake_scan_for_unused_images

        self.image_scanner.start(self.timeout,
                                 self.VM_SCAN_RATE,
                                 self.IMAGE_MARK_RATE)

        self.image_scanner.wait_for_task_end()

        assert_that(self.image_scanner.get_state() is
                    DatastoreImageScanner.State.IDLE)

        scan_vms_for_active_images.assert_called_once_with(self.image_scanner, self.DATASTORE_ID)
        scan_for_unused_images.assert_called_once_with(self.image_scanner, self.DATASTORE_ID)

    @patch("host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_for_unused_images")
    @patch("host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_vms_for_active_images")
    def test_stop(self, scan_vms_for_active_images, scan_for_unused_images):
        assert_that(self.image_scanner.get_state() is
                    DatastoreImageScanner.State.IDLE)

        scan_vms_for_active_images.side_effect = self.fake_scan_vms_for_active_images
        scan_for_unused_images.side_effect = self.fake_scan_for_unused_images

        self.wait_at_the_end_of_scan = True
        self.image_scanner.start(self.timeout,
                                 self.VM_SCAN_RATE,
                                 self.IMAGE_MARK_RATE)

        self.wait_for_runner()
        self.image_scanner.stop()
        self.signal_runner()

        self.image_scanner.wait_for_task_end()

        assert_that(self.image_scanner.get_state() is
                    DatastoreImageScanner.State.IDLE)
        exception = self.image_scanner.get_exception()
        assert_that(isinstance(exception, TaskTerminated) is True)

        scan_vms_for_active_images.assert_called_with(self.image_scanner, self.DATASTORE_ID)
        assert_that(scan_for_unused_images.called is False)

    @patch("host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_for_unused_images")
    @patch("host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_vms_for_active_images")
    def test_timeout(self, scan_vms_for_active_images, scan_for_unused_images):
        assert_that(self.image_scanner.get_state() is
                    DatastoreImageScanner.State.IDLE)

        scan_vms_for_active_images.side_effect = self.fake_scan_vms_for_active_images
        scan_for_unused_images.side_effect = self.fake_scan_for_unused_images

        self.timeout = 1
        self.wait_at_the_end_of_scan = True
        self.image_scanner.start(self.timeout,
                                 self.VM_SCAN_RATE,
                                 self.IMAGE_MARK_RATE)

        self.wait_for_runner()
        time.sleep(2)
        self.signal_runner()

        self.image_scanner.wait_for_task_end()

        assert_that(self.image_scanner.get_state() is
                    DatastoreImageScanner.State.IDLE)
        exception = self.image_scanner.get_exception()
        assert_that(isinstance(exception, TaskTimeout) is True)

        scan_vms_for_active_images.assert_called_with(self.image_scanner, self.DATASTORE_ID)
        assert_that(scan_for_unused_images.called is False)

    def test_parse_vmdk(self):
        base_dir = os.path.dirname(__file__)
        test_dir = os.path.join(base_dir, "../test_files", "vm_good")
        vmdk_pathname = os.path.join(test_dir, 'good.vmdk')
        dictionary = DatastoreImageScannerTaskRunner._parse_vmdk(vmdk_pathname)
        assert_that(dictionary["version"] == "1")
        assert_that(dictionary["encoding"] == "UTF-8")
        assert_that(dictionary["CID"] == "fffffffe")
        assert_that(dictionary["parentCID"] == "fffffffe")
        assert_that(dictionary["isNativeSnapshot"] == "no")
        assert_that(dictionary["createType"] == "vmfsSparse")
        assert_that(dictionary["parentFileNameHint"] ==
                    "/vmfs/volumes/555ca9f8-9f24fa2c-41c1-0025b5414043/"
                    "image_92e62599-6689-4a8f-ba2a-633914b5048e/92e"
                    "62599-6689-4a8f-ba2a-633914b5048e.vmdk")

    def fake_scan_vms_for_active_images(self, image_scanner, datastore):
        assert_that(image_scanner.datastore_id is self.DATASTORE_ID)
        assert_that(datastore is self.DATASTORE_ID)
        assert_that(self.image_scanner.get_state() is
                    DatastoreImageScanner.State.VM_SCAN)
        assert_that(image_scanner.vm_scan_rate is self.VM_SCAN_RATE)
        if self.wait_at_the_end_of_scan:
            # Wait here until the main thread wants to continue
            self.signal_main()
            self.wait_for_main()
        return dict()

    def fake_scan_for_unused_images(self, image_scanner, datastore):
        assert_that(image_scanner.datastore_id is self.DATASTORE_ID)
        assert_that(datastore is self.DATASTORE_ID)
        assert_that(self.image_scanner.get_state() is
                    DatastoreImageScanner.State.IMAGE_MARK)
        assert_that(image_scanner.image_mark_rate is self.IMAGE_MARK_RATE)
        return dict()

    def wait_for_runner(self):
        self.synchronizer.wait_for_runner()

    def signal_runner(self):
        self.synchronizer.signal_runner()

    def wait_for_main(self):
        self.synchronizer.wait_for_main()

    def signal_main(self):
        self.synchronizer.signal_main()