def setUp(self):
self.image_manager = MagicMock()
self.image_manager.delete_unused.side_effect = self.fake_delete_unused_images
self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID)
self.synchronizer = TestSynchronizer()
self.timeout = self.TIMEOUT
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "image_")
self.dir0 = dir0
# Image dir with correct timestamp file
image_id_1 = str(uuid.uuid4())
dir1 = self.create_dir(image_id_1)
open(os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
# Image dir without the correct timestamp file
image_id_2 = str(uuid.uuid4())
dir2 = self.create_dir(image_id_2)
# Image dir with correct timestamp file
# and with tombstone file
image_id_3 = str(uuid.uuid4())
dir3 = self.create_dir(image_id_3)
open(os.path.join(dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
self.image_ids = ["", image_id_1, image_id_2, image_id_3]
self.image_dirs = ["", dir1, dir2, dir3]
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.vim_client = MagicMock()
self.vim_client.delete_file.side_effect = self.patched_delete
self.image_manager = ImageManager(self.vim_client, MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID)
self.deleted = False
self.marker_unlinked = False
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = str(uuid.uuid4())
self.image_id_1 = image_id_1
self.image_id_2 = image_id_2
self.image_id_3 = image_id_3
self.image_id_4 = image_id_4
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
self.marker_file_content_time = 0
self.timestamp_file_mod_time = 0
# Create a good image vmdk under image_id_1, also create a valid image marker file and a valid timestamp file
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
marker_filename = os.path.join(dir1, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_2, create timestamp but no image marker file,
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir2, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
# Create a good image vmdk under image_id_3, create image_marker file but no timestamp file
vmdk_filename = image_id_3 + ".vmdk"
open(os.path.join(dir3, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir3, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_4, create image_marker file, renamed timestamp file
# but no timestamp file
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir4, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.delete_count = 0
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = "invalid_image_id"
self.image_ids = ["*", image_id_1, image_id_2, image_id_3, image_id_4]
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
# Create a vmdk under "im", since the image_id is not a valid uuid it should be skipped
open(os.path.join(self.test_dir, "image_im.vmdk"), 'w').close()
# Create a good image vmdk under image_id_1 but no image marker file, this should not be deleted
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
# Create a good image vmdk under image_id_2, also create an unused image marker file,
# image_id_2 should be deleted
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
open(os.path.join(dir2, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a marker file under dir3 but no vmdk file. It should be deleted as well
open(os.path.join(dir3, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a vmdk under an invalid image directory, also create a marker file. Since the image_id
# is not valid it should not be deleted
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
open(os.path.join(dir4, self.IMAGE_MARKER_FILENAME), 'w').close()
def __init__(self, datastore_manager, image_manager, vm_manager):
self.logger = logging.getLogger(__name__)
self.datastore_manager = datastore_manager
self.datastore_image_scanners = dict()
self.datastore_image_sweepers = dict()
for datastore_id in self.datastore_manager.get_datastore_ids():
self.logger.info("IMAGE SCANNER: adding datastore: %s" %
datastore_id)
self.datastore_image_scanners[datastore_id] = \
DatastoreImageScanner(image_manager,
vm_manager,
datastore_id)
self.datastore_image_sweepers[datastore_id] = \
DatastoreImageSweeper(image_manager,
datastore_id)
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID)
self.delete_count = 0
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = "invalid_image_id"
self.image_ids = ["*", image_id_1, image_id_2, image_id_3, image_id_4]
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
# Create a vmdk under "im", since the image_id is not a valid uuid it should be skipped
open(os.path.join(self.test_dir, "image_im.vmdk"), 'w').close()
# Create a good image vmdk under image_id_1 but no image marker file, this should not be deleted
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
# Create a good image vmdk under image_id_2, also create an unused image marker file,
# image_id_2 should be deleted
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
open(os.path.join(dir2, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a marker file under dir3 but no vmdk file. It should be deleted as well
open(os.path.join(dir3, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a vmdk under an invalid image directory, also create a marker file. Since the image_id
# is not valid it should not be deleted
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
open(os.path.join(dir4, self.IMAGE_MARKER_FILENAME), 'w').close()
class ImageSweeperTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
BASE_TEMP_DIR = "image_sweeper"
IMAGE_MARKER_FILENAME = ImageManager.UNUSED_IMAGE_MARKER_FILE_NAME
IMAGE_TIMESTAMP_FILENAME = ImageManager.IMAGE_TIMESTAMP_FILE_NAME
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID)
self.delete_count = 0
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = "invalid_image_id"
self.image_ids = ["*", image_id_1, image_id_2, image_id_3, image_id_4]
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
# Create a vmdk under "im", since the image_id is not a valid uuid it should be skipped
open(os.path.join(self.test_dir, "image_im.vmdk"), 'w').close()
# Create a good image vmdk under image_id_1 but no image marker file, this should not be deleted
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
# Create a good image vmdk under image_id_2, also create an unused image marker file,
# image_id_2 should be deleted
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
open(os.path.join(dir2, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a marker file under dir3 but no vmdk file. It should be deleted as well
open(os.path.join(dir3, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a vmdk under an invalid image directory, also create a marker file. Since the image_id
# is not valid it should not be deleted
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
open(os.path.join(dir4, self.IMAGE_MARKER_FILENAME), 'w').close()
def tearDown(self):
shutil.rmtree(self.test_dir, True)
@parameterized.expand([
# image_id, target image_id, delete count
(1, 0), # 1 image, no marker file, 0 delete
(2, 1), # 1 image, marker file, 1 delete
(3, 1), # 0 image, marker file, 1 delete
])
@patch("host.image.image_manager.ImageManager.delete_image")
@patch("host.image.image_sweeper.DatastoreImageSweeper.is_stopped", return_value=False)
def test_image_sweeper(self, target_image_id_index, deleted_count, is_stopped, delete_image):
delete_image.side_effect = self.patched_delete_image
self.image_sweeper.image_sweep_rate = 60000
target_image_id = self.image_ids[target_image_id_index]
self.image_sweeper.set_target_images([target_image_id])
deleted_list = self.image_sweeper._task_runner._delete_unused_images(self.image_sweeper, self.test_dir)
assert_that(len(deleted_list) is deleted_count)
assert_that(self.delete_count is deleted_count)
@patch("host.image.image_manager.ImageManager.delete_image")
@patch("host.image.image_sweeper.DatastoreImageSweeper.is_stopped", return_value=False)
def test_image_sweeper_bad_root(self, is_stopped, delete_image):
delete_image.side_effect = self.patched_delete_image
self.image_sweeper.image_sweep_rate = 60000
self.image_sweeper.set_target_images(["image_id_5"])
dictionary = self.image_sweeper._task_runner._delete_unused_images(self.image_sweeper, self.test_dir)
assert_that(len(dictionary) is 0)
assert_that(self.delete_count is 0)
def patched_delete_image(self, datastore_id, image_id, grace_period):
self.delete_count += 1
return True
class ImageSweeperTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
IMAGE_SWEEP_RATE = 12
TIMEOUT = 6000
# This test uses two threads, the main thread is used to check
# the state inside the state machine at different stages. The
# second thread, executes the steps in sequence and calls out
# the main method:
# delete_unused()
# These methods have been patched to synchronize with the main
# thread.
def setUp(self):
self.image_manager = MagicMock()
self.image_manager.delete_unused.side_effect = self.fake_delete_unused_images
self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID)
self.synchronizer = TestSynchronizer()
self.timeout = self.TIMEOUT
def tearDown(self):
self.image_sweeper.stop()
@patch("host.image.image_sweeper.DatastoreImageSweeperTaskRunner._delete_unused_images")
def test_lifecycle(self, delete_unused_images):
assert_that(self.image_sweeper.get_state() is DatastoreImageSweeper.State.IDLE)
delete_unused_images.side_effect = self.fake_delete_unused_images
self.image_sweeper.start(self.timeout, list(), self.IMAGE_SWEEP_RATE)
self.image_sweeper.wait_for_task_end()
assert_that(self.image_sweeper.get_state() is DatastoreImageSweeper.State.IDLE)
delete_unused_images.assert_called_once_with(self.image_sweeper, "/vmfs/volumes/DS01")
deleted_images, _ = self.image_sweeper.get_deleted_images()
assert_that(len(deleted_images) is 3)
def fake_delete_unused_images(self, image_sweeper, datastore_root):
assert_that(image_sweeper.datastore_id is self.DATASTORE_ID)
assert_that(self.image_sweeper.get_state() is DatastoreImageSweeper.State.IMAGE_SWEEP)
assert_that(image_sweeper.image_sweep_rate is self.IMAGE_SWEEP_RATE)
deleted_images = list()
deleted_images.append("image_id_1")
deleted_images.append("image_id_2")
deleted_images.append("image_id_3")
return deleted_images
def wait_for_runner(self):
self.synchronizer.wait_for_runner()
def signal_runner(self):
self.synchronizer.signal_runner()
def wait_for_main(self):
self.synchronizer.wait_for_main()
def signal_main(self):
self.synchronizer.signal_main()
class ImageSweeperTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
BASE_TEMP_DIR = "image_sweeper"
IMAGE_MARKER_FILENAME = ImageManager.UNUSED_IMAGE_MARKER_FILE_NAME
IMAGE_TIMESTAMP_FILENAME = ImageManager.IMAGE_TIMESTAMP_FILE_NAME
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID)
self.delete_count = 0
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = "invalid_image_id"
self.image_ids = ["*", image_id_1, image_id_2, image_id_3, image_id_4]
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
# Create a vmdk under "im", since the image_id is not a valid uuid it should be skipped
open(os.path.join(self.test_dir, "image_im.vmdk"), 'w').close()
# Create a good image vmdk under image_id_1 but no image marker file, this should not be deleted
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
# Create a good image vmdk under image_id_2, also create an unused image marker file,
# image_id_2 should be deleted
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
open(os.path.join(dir2, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a marker file under dir3 but no vmdk file. It should be deleted as well
open(os.path.join(dir3, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a vmdk under an invalid image directory, also create a marker file. Since the image_id
# is not valid it should not be deleted
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
open(os.path.join(dir4, self.IMAGE_MARKER_FILENAME), 'w').close()
def tearDown(self):
shutil.rmtree(self.test_dir, True)
@parameterized.expand([
# image_id, target image_id, delete count
(1, 0), # 1 image, no marker file, 0 delete
(2, 1), # 1 image, marker file, 1 delete
(3, 1), # 0 image, marker file, 1 delete
])
@patch("host.image.image_manager.ImageManager.delete_image")
@patch("host.image.image_sweeper.DatastoreImageSweeper.is_stopped", return_value=False)
def test_image_sweeper(self, target_image_id_index, deleted_count, is_stopped, delete_image):
delete_image.side_effect = self.patched_delete_image
self.image_sweeper.image_sweep_rate = 60000
target_image_id = self.image_ids[target_image_id_index]
self.image_sweeper.set_target_images([target_image_id])
deleted_list = self.image_sweeper._task_runner._delete_unused_images(self.image_sweeper, self.test_dir)
assert_that(len(deleted_list) is deleted_count)
assert_that(self.delete_count is deleted_count)
@patch("host.image.image_manager.ImageManager.delete_image")
@patch("host.image.image_sweeper.DatastoreImageSweeper.is_stopped", return_value=False)
def test_image_sweeper_bad_root(self, is_stopped, delete_image):
delete_image.side_effect = self.patched_delete_image
self.image_sweeper.image_sweep_rate = 60000
self.image_sweeper.set_target_images(["image_id_5"])
dictionary = self.image_sweeper._task_runner._delete_unused_images(self.image_sweeper, self.test_dir)
assert_that(len(dictionary) is 0)
assert_that(self.delete_count is 0)
def patched_delete_image(self, datastore_id, image_id, grace_period):
self.delete_count += 1
return True