def setUp(self):
self.image_manager = MagicMock()
self.vm_manager = MagicMock()
self.image_scanner = DatastoreImageScanner(self.image_manager,
self.vm_manager,
self.DATASTORE_ID)
self.synchronizer = TestSynchronizer()
self.wait_at_the_end_of_scan = False
self.raise_exception = False
self.timeout = self.TIMEOUT
def setUp(self):
self.image_manager = MagicMock()
self.vm_manager = MagicMock()
self.image_scanner = DatastoreImageScanner(
self.image_manager,
self.vm_manager,
self.DATASTORE_ID)
self.synchronizer = TestSynchronizer()
self.wait_at_the_end_of_scan = False
self.raise_exception = False
self.timeout = self.TIMEOUT
def setUp(self):
# Create VM manager
self.vim_client = VimClient(auto_sync=False)
self.vim_client._content = MagicMock()
self.vim_client.wait_for_task = MagicMock()
self.vm_manager = VmManager(self.vim_client, MagicMock())
services.register(ServiceName.AGENT_CONFIG, MagicMock())
# Set up test files
self.base_dir = os.path.dirname(__file__)
self.test_dir = os.path.join(self.base_dir, "../test_files")
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.image_scanner = DatastoreImageScanner(self.image_manager, self.vm_manager, self.DATASTORE_ID)
self.write_count = 0
def __init__(self, datastore_manager, image_manager, vm_manager):
self.logger = logging.getLogger(__name__)
self.datastore_manager = datastore_manager
self.datastore_image_scanners = dict()
self.datastore_image_sweepers = dict()
for datastore_id in self.datastore_manager.get_datastore_ids():
self.logger.info("IMAGE SCANNER: adding datastore: %s" %
datastore_id)
self.datastore_image_scanners[datastore_id] = \
DatastoreImageScanner(image_manager,
vm_manager,
datastore_id)
self.datastore_image_sweepers[datastore_id] = \
DatastoreImageSweeper(image_manager,
datastore_id)
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_scanner = DatastoreImageScanner(self.image_manager,
self.vm_manager,
self.DATASTORE_ID)
self.write_count = 0
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = "invalid_image_id"
self.image_ids = ["*", image_id_1, image_id_2, image_id_3, image_id_4]
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
# Create a vmdk under "im", since the image_id is not a valid uuid it should be skipped
open(os.path.join(self.test_dir, "image_im.vmdk"), 'w').close()
# Create a good image vmdk under image_id_1, the name of the vmdk matches the directory
# that contains it so this is a valid image to remove
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
# Create a good image vmdk under image_id_2, also create an unused image marker file,
# image_id_2 should also be included in the list of images to remove
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
open(
os.path.join(dir2,
self.image_manager.UNUSED_IMAGE_MARKER_FILE_NAME),
'w').close()
# Don't create anything under directory dir3. it should still mark the image as deletable
# Create a vmdk under an invalid image directory, since the image id is not valid it
# should not mark it for deletion
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
class ImageScannerTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
VM_SCAN_RATE = 11
IMAGE_MARK_RATE = 12
TIMEOUT = 6000
# This test uses two threads, the main thread is used to check
# the state inside the state machine at different stages. The
# second thread, ImageScannerThread, executes the steps in
# sequence and calls out the two main methods:
# get_active_images() and mark_unused()
# These methods have been patched to synchronize with the main
# thread.
def setUp(self):
self.image_manager = MagicMock()
self.vm_manager = MagicMock()
self.image_scanner = DatastoreImageScanner(self.image_manager,
self.vm_manager,
self.DATASTORE_ID)
self.synchronizer = TestSynchronizer()
self.wait_at_the_end_of_scan = False
self.raise_exception = False
self.timeout = self.TIMEOUT
def tearDown(self):
self.image_scanner.stop()
@patch(
"host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_for_unused_images"
)
@patch(
"host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_vms_for_active_images"
)
def test_lifecycle(self, scan_vms_for_active_images,
scan_for_unused_images):
assert_that(
self.image_scanner.get_state() is DatastoreImageScanner.State.IDLE)
scan_vms_for_active_images.side_effect = self.fake_scan_vms_for_active_images
scan_for_unused_images.side_effect = self.fake_scan_for_unused_images
self.image_scanner.start(self.timeout, self.VM_SCAN_RATE,
self.IMAGE_MARK_RATE)
self.image_scanner.wait_for_task_end()
assert_that(
self.image_scanner.get_state() is DatastoreImageScanner.State.IDLE)
scan_vms_for_active_images.assert_called_once_with(
self.image_scanner, self.DATASTORE_ID)
scan_for_unused_images.assert_called_once_with(self.image_scanner,
self.DATASTORE_ID)
@patch(
"host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_for_unused_images"
)
@patch(
"host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_vms_for_active_images"
)
def test_stop(self, scan_vms_for_active_images, scan_for_unused_images):
assert_that(
self.image_scanner.get_state() is DatastoreImageScanner.State.IDLE)
scan_vms_for_active_images.side_effect = self.fake_scan_vms_for_active_images
scan_for_unused_images.side_effect = self.fake_scan_for_unused_images
self.wait_at_the_end_of_scan = True
self.image_scanner.start(self.timeout, self.VM_SCAN_RATE,
self.IMAGE_MARK_RATE)
self.wait_for_runner()
self.image_scanner.stop()
self.signal_runner()
self.image_scanner.wait_for_task_end()
assert_that(
self.image_scanner.get_state() is DatastoreImageScanner.State.IDLE)
exception = self.image_scanner.get_exception()
assert_that(isinstance(exception, TaskTerminated) is True)
scan_vms_for_active_images.assert_called_with(self.image_scanner,
self.DATASTORE_ID)
assert_that(scan_for_unused_images.called is False)
@patch(
"host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_for_unused_images"
)
@patch(
"host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_vms_for_active_images"
)
def test_timeout(self, scan_vms_for_active_images, scan_for_unused_images):
assert_that(
self.image_scanner.get_state() is DatastoreImageScanner.State.IDLE)
scan_vms_for_active_images.side_effect = self.fake_scan_vms_for_active_images
scan_for_unused_images.side_effect = self.fake_scan_for_unused_images
self.timeout = 1
self.wait_at_the_end_of_scan = True
self.image_scanner.start(self.timeout, self.VM_SCAN_RATE,
self.IMAGE_MARK_RATE)
self.wait_for_runner()
time.sleep(2)
self.signal_runner()
self.image_scanner.wait_for_task_end()
assert_that(
self.image_scanner.get_state() is DatastoreImageScanner.State.IDLE)
exception = self.image_scanner.get_exception()
assert_that(isinstance(exception, TaskTimeout) is True)
scan_vms_for_active_images.assert_called_with(self.image_scanner,
self.DATASTORE_ID)
assert_that(scan_for_unused_images.called is False)
def test_parse_vmdk(self):
base_dir = os.path.dirname(__file__)
test_dir = os.path.join(base_dir, "../test_files", "vm_good")
vmdk_pathname = os.path.join(test_dir, 'good.vmdk')
dictionary = DatastoreImageScannerTaskRunner._parse_vmdk(vmdk_pathname)
assert_that(dictionary["version"] == "1")
assert_that(dictionary["encoding"] == "UTF-8")
assert_that(dictionary["CID"] == "fffffffe")
assert_that(dictionary["parentCID"] == "fffffffe")
assert_that(dictionary["isNativeSnapshot"] == "no")
assert_that(dictionary["createType"] == "vmfsSparse")
assert_that(dictionary["parentFileNameHint"] ==
"/vmfs/volumes/555ca9f8-9f24fa2c-41c1-0025b5414043/"
"image_92e62599-6689-4a8f-ba2a-633914b5048e/92e"
"62599-6689-4a8f-ba2a-633914b5048e.vmdk")
def fake_scan_vms_for_active_images(self, image_scanner, datastore):
assert_that(image_scanner.datastore_id is self.DATASTORE_ID)
assert_that(datastore is self.DATASTORE_ID)
assert_that(self.image_scanner.get_state() is
DatastoreImageScanner.State.VM_SCAN)
assert_that(image_scanner.vm_scan_rate is self.VM_SCAN_RATE)
if self.wait_at_the_end_of_scan:
# Wait here until the main thread wants to continue
self.signal_main()
self.wait_for_main()
return dict()
def fake_scan_for_unused_images(self, image_scanner, datastore):
assert_that(image_scanner.datastore_id is self.DATASTORE_ID)
assert_that(datastore is self.DATASTORE_ID)
assert_that(self.image_scanner.get_state() is
DatastoreImageScanner.State.IMAGE_MARK)
assert_that(image_scanner.image_mark_rate is self.IMAGE_MARK_RATE)
return dict()
def wait_for_runner(self):
self.synchronizer.wait_for_runner()
def signal_runner(self):
self.synchronizer.signal_runner()
def wait_for_main(self):
self.synchronizer.wait_for_main()
def signal_main(self):
self.synchronizer.signal_main()
class ImageScannerTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
VM_SCAN_RATE = 11
IMAGE_MARK_RATE = 12
TIMEOUT = 6000
# This test uses two threads, the main thread is used to check
# the state inside the state machine at different stages. The
# second thread, ImageScannerThread, executes the steps in
# sequence and calls out the two main methods:
# get_active_images() and mark_unused()
# These methods have been patched to synchronize with the main
# thread.
def setUp(self):
self.image_manager = MagicMock()
self.vm_manager = MagicMock()
self.image_scanner = DatastoreImageScanner(
self.image_manager,
self.vm_manager,
self.DATASTORE_ID)
self.synchronizer = TestSynchronizer()
self.wait_at_the_end_of_scan = False
self.raise_exception = False
self.timeout = self.TIMEOUT
def tearDown(self):
self.image_scanner.stop()
@patch("host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_for_unused_images")
@patch("host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_vms_for_active_images")
def test_lifecycle(self, scan_vms_for_active_images, scan_for_unused_images):
assert_that(self.image_scanner.get_state() is
DatastoreImageScanner.State.IDLE)
scan_vms_for_active_images.side_effect = self.fake_scan_vms_for_active_images
scan_for_unused_images.side_effect = self.fake_scan_for_unused_images
self.image_scanner.start(self.timeout,
self.VM_SCAN_RATE,
self.IMAGE_MARK_RATE)
self.image_scanner.wait_for_task_end()
assert_that(self.image_scanner.get_state() is
DatastoreImageScanner.State.IDLE)
scan_vms_for_active_images.assert_called_once_with(self.image_scanner, self.DATASTORE_ID)
scan_for_unused_images.assert_called_once_with(self.image_scanner, self.DATASTORE_ID)
@patch("host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_for_unused_images")
@patch("host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_vms_for_active_images")
def test_stop(self, scan_vms_for_active_images, scan_for_unused_images):
assert_that(self.image_scanner.get_state() is
DatastoreImageScanner.State.IDLE)
scan_vms_for_active_images.side_effect = self.fake_scan_vms_for_active_images
scan_for_unused_images.side_effect = self.fake_scan_for_unused_images
self.wait_at_the_end_of_scan = True
self.image_scanner.start(self.timeout,
self.VM_SCAN_RATE,
self.IMAGE_MARK_RATE)
self.wait_for_runner()
self.image_scanner.stop()
self.signal_runner()
self.image_scanner.wait_for_task_end()
assert_that(self.image_scanner.get_state() is
DatastoreImageScanner.State.IDLE)
exception = self.image_scanner.get_exception()
assert_that(isinstance(exception, TaskTerminated) is True)
scan_vms_for_active_images.assert_called_with(self.image_scanner, self.DATASTORE_ID)
assert_that(scan_for_unused_images.called is False)
@patch("host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_for_unused_images")
@patch("host.image.image_scanner.DatastoreImageScannerTaskRunner._scan_vms_for_active_images")
def test_timeout(self, scan_vms_for_active_images, scan_for_unused_images):
assert_that(self.image_scanner.get_state() is
DatastoreImageScanner.State.IDLE)
scan_vms_for_active_images.side_effect = self.fake_scan_vms_for_active_images
scan_for_unused_images.side_effect = self.fake_scan_for_unused_images
self.timeout = 1
self.wait_at_the_end_of_scan = True
self.image_scanner.start(self.timeout,
self.VM_SCAN_RATE,
self.IMAGE_MARK_RATE)
self.wait_for_runner()
time.sleep(2)
self.signal_runner()
self.image_scanner.wait_for_task_end()
assert_that(self.image_scanner.get_state() is
DatastoreImageScanner.State.IDLE)
exception = self.image_scanner.get_exception()
assert_that(isinstance(exception, TaskTimeout) is True)
scan_vms_for_active_images.assert_called_with(self.image_scanner, self.DATASTORE_ID)
assert_that(scan_for_unused_images.called is False)
def test_parse_vmdk(self):
base_dir = os.path.dirname(__file__)
test_dir = os.path.join(base_dir, "../test_files", "vm_good")
vmdk_pathname = os.path.join(test_dir, 'good.vmdk')
dictionary = DatastoreImageScannerTaskRunner._parse_vmdk(vmdk_pathname)
assert_that(dictionary["version"] == "1")
assert_that(dictionary["encoding"] == "UTF-8")
assert_that(dictionary["CID"] == "fffffffe")
assert_that(dictionary["parentCID"] == "fffffffe")
assert_that(dictionary["isNativeSnapshot"] == "no")
assert_that(dictionary["createType"] == "vmfsSparse")
assert_that(dictionary["parentFileNameHint"] ==
"/vmfs/volumes/555ca9f8-9f24fa2c-41c1-0025b5414043/"
"image_92e62599-6689-4a8f-ba2a-633914b5048e/92e"
"62599-6689-4a8f-ba2a-633914b5048e.vmdk")
def fake_scan_vms_for_active_images(self, image_scanner, datastore):
assert_that(image_scanner.datastore_id is self.DATASTORE_ID)
assert_that(datastore is self.DATASTORE_ID)
assert_that(self.image_scanner.get_state() is
DatastoreImageScanner.State.VM_SCAN)
assert_that(image_scanner.vm_scan_rate is self.VM_SCAN_RATE)
if self.wait_at_the_end_of_scan:
# Wait here until the main thread wants to continue
self.signal_main()
self.wait_for_main()
return dict()
def fake_scan_for_unused_images(self, image_scanner, datastore):
assert_that(image_scanner.datastore_id is self.DATASTORE_ID)
assert_that(datastore is self.DATASTORE_ID)
assert_that(self.image_scanner.get_state() is
DatastoreImageScanner.State.IMAGE_MARK)
assert_that(image_scanner.image_mark_rate is self.IMAGE_MARK_RATE)
return dict()
def wait_for_runner(self):
self.synchronizer.wait_for_runner()
def signal_runner(self):
self.synchronizer.signal_runner()
def wait_for_main(self):
self.synchronizer.wait_for_main()
def signal_main(self):
self.synchronizer.signal_main()