def setUp(self):
self.image_manager = MagicMock()
self.image_manager.delete_unused.side_effect = self.fake_delete_unused_images
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.synchronizer = TestSynchronizer()
self.timeout = self.TIMEOUT
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, "images/im")
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = "invalid_image_id"
self.image_ids = ["", image_id_1, image_id_2, image_id_3, image_id_4]
dir1 = os.path.join(dir0, image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(dir0, image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(dir0, image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(dir0, image_id_4)
os.makedirs(dir4)
# Create a vmdk under "im", since the image_id is
# not a valid uuid it should be skipped
open(os.path.join(dir0, "im.vmdk"), 'w').close()
# Create a good image vmdk under image_id_1 but
# no image marker file, this should not be deleted
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
# Create a good image vmdk under image_id_2, also create
# an unused image marker file, image_id_2 should be
# deleted
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
open(os.path.join(dir2, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a marker file under dir3 but no
# vmdk file. It should be deleted as well
open(os.path.join(dir3, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a vmdk under an invalid image directory,
# also create a marker file. Since the image_id
# is not valid it should not be deleted
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
open(os.path.join(dir4, self.IMAGE_MARKER_FILENAME), 'w').close()
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "image_")
self.dir0 = dir0
# Image dir with correct timestamp file
image_id_1 = str(uuid.uuid4())
dir1 = self.create_dir(image_id_1)
open(os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
# Image dir without the correct timestamp file
image_id_2 = str(uuid.uuid4())
dir2 = self.create_dir(image_id_2)
# Image dir with correct timestamp file
# and with tombstone file
image_id_3 = str(uuid.uuid4())
dir3 = self.create_dir(image_id_3)
open(os.path.join(
dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
self.image_ids = ["", image_id_1, image_id_2, image_id_3]
self.image_dirs = ["", dir1, dir2, dir3]
def setUp(self):
self.image_manager = MagicMock()
self.image_manager.delete_unused.side_effect = self.fake_delete_unused_images
self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID)
self.synchronizer = TestSynchronizer()
self.timeout = self.TIMEOUT
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, "images/im")
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = "invalid_image_id"
self.image_ids = ["", image_id_1, image_id_2,
image_id_3, image_id_4]
dir1 = os.path.join(dir0, image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(dir0, image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(dir0, image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(dir0, image_id_4)
os.makedirs(dir4)
# Create a vmdk under "im", since the image_id is
# not a valid uuid it should be skipped
open(os.path.join(dir0, "im.vmdk"), 'w').close()
# Create a good image vmdk under image_id_1 but
# no image marker file, this should not be deleted
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
# Create a good image vmdk under image_id_2, also create
# an unused image marker file, image_id_2 should be
# deleted
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
open(os.path.join(
dir2, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a marker file under dir3 but no
# vmdk file. It should be deleted as well
open(os.path.join(
dir3, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a vmdk under an invalid image directory,
# also create a marker file. Since the image_id
# is not valid it should not be deleted
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
open(os.path.join(
dir4, self.IMAGE_MARKER_FILENAME), 'w').close()
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
self.gc_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.deleted = False
self.marker_unlinked = False
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = str(uuid.uuid4())
self.image_id_1 = image_id_1
self.image_id_2 = image_id_2
self.image_id_3 = image_id_3
self.image_id_4 = image_id_4
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
self.marker_file_content_time = 0
self.timestamp_file_mod_time = 0
self.renamed_timestamp_file_mod_time = 0
# Create a good image vmdk under image_id_1,
# also create a valid image marker file
# and a valid timestamp file
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
marker_filename = os.path.join(dir1, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_2,
# create timestamp but no image marker file,
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir2, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
# Create a good image vmdk under image_id_3,
# create image_marker file but no timestamp
# and no renamed timestamp file
vmdk_filename = image_id_3 + ".vmdk"
open(os.path.join(dir3, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir3, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_4,
# create image_marker file, renamed timestamp file
# but no timestamp file
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir4, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
timestamp_filename = os.path.join(dir4, self.IMAGE_TIMESTAMP_FILENAME)
renamed_timestamp_filename = timestamp_filename + self.IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX
open(renamed_timestamp_filename, 'w').close()
class ImageSweeperTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
BASE_TEMP_DIR = "image_sweeper"
IMAGE_MARKER_FILENAME = EsxImageManager.IMAGE_MARKER_FILE_NAME
IMAGE_TIMESTAMP_FILENAME = EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID)
self.delete_count = 0
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = "invalid_image_id"
self.image_ids = ["*", image_id_1, image_id_2, image_id_3, image_id_4]
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
# Create a vmdk under "im", since the image_id is
# not a valid uuid it should be skipped
open(os.path.join(self.test_dir, "image_im.vmdk"), 'w').close()
# Create a good image vmdk under image_id_1 but
# no image marker file, this should not be deleted
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
# Create a good image vmdk under image_id_2, also create
# an unused image marker file, image_id_2 should be
# deleted
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
open(os.path.join(
dir2, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a marker file under dir3 but no
# vmdk file. It should be deleted as well
open(os.path.join(
dir3, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a vmdk under an invalid image directory,
# also create a marker file. Since the image_id
# is not valid it should not be deleted
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
open(os.path.join(
dir4, self.IMAGE_MARKER_FILENAME), 'w').close()
def tearDown(self):
shutil.rmtree(self.test_dir, True)
@parameterized.expand([
# image_id, target image_id, delete count
(1, 0), # 1 image, no marker file, 0 delete
(2, 1), # 1 image, marker file, 1 delete
(3, 1), # 0 image, marker file, 1 delete
])
@patch("host.hypervisor.esx.image_manager.EsxImageManager._delete_single_image")
@patch("host.hypervisor.image_sweeper.DatastoreImageSweeper.is_stopped", return_value=False)
def test_image_sweeper(self, target_image_id_index, deleted_count, is_stopped, delete_single_image):
delete_single_image.side_effect = self.patched_delete_single_image
self.image_sweeper.image_sweep_rate = 60000
target_image_id = self.image_ids[target_image_id_index]
self.image_sweeper.set_target_images([target_image_id])
deleted_list = self.image_sweeper._task_runner._delete_unused_images(self.image_sweeper, self.test_dir)
assert_that(len(deleted_list) is deleted_count)
assert_that(self.delete_count is deleted_count)
@patch("host.hypervisor.esx.image_manager.EsxImageManager._delete_single_image")
@patch("host.hypervisor.image_sweeper.DatastoreImageSweeper.is_stopped", return_value=False)
def test_image_sweeper_bad_root(self, is_stopped, delete_single_image):
delete_single_image.side_effect = self.patched_delete_single_image
self.image_sweeper.image_sweep_rate = 60000
self.image_sweeper.set_target_images(["image_id_5"])
dictionary = self.image_sweeper._task_runner._delete_unused_images(self.image_sweeper, self.test_dir)
assert_that(len(dictionary) is 0)
assert_that(self.delete_count is 0)
def patched_delete_single_image(self, image_sweeper, pathname, image_id):
self.delete_count += 1
return True
def patched_image_directory_path(datastore, image_id):
return "abc" + image_id
class ImageSweeperTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
BASE_TEMP_DIR = "image_sweeper"
IMAGE_MARKER_FILENAME = EsxImageManager.IMAGE_MARKER_FILE_NAME
IMAGE_TIMESTAMP_FILENAME = EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME
IMAGE_TOMBSTONE_FILE_NAME = EsxImageManager.IMAGE_TOMBSTONE_FILE_NAME
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, "images/im")
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = "invalid_image_id"
self.image_ids = ["", image_id_1, image_id_2,
image_id_3, image_id_4]
dir1 = os.path.join(dir0, image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(dir0, image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(dir0, image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(dir0, image_id_4)
os.makedirs(dir4)
# Create a vmdk under "im", since the image_id is
# not a valid uuid it should be skipped
open(os.path.join(dir0, "im.vmdk"), 'w').close()
# Create a good image vmdk under image_id_1 but
# no image marker file, this should not be deleted
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
# Create a good image vmdk under image_id_2, also create
# an unused image marker file, image_id_2 should be
# deleted
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
open(os.path.join(
dir2, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a marker file under dir3 but no
# vmdk file. It should be deleted as well
open(os.path.join(
dir3, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a vmdk under an invalid image directory,
# also create a marker file. Since the image_id
# is not valid it should not be deleted
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
open(os.path.join(
dir4, self.IMAGE_MARKER_FILENAME), 'w').close()
def tearDown(self):
shutil.rmtree(self.test_dir, True)
@parameterized.expand([
# image_id, target image_id, delete count
(1, 1, 0), # 1 image, no marker file, 0 delete
(2, 2, 1), # 1 image, marker file, 1 delete
(3, 3, 1), # 0 image, marker file, 1 delete
(4, 4, 0), # 1 image, marker file, 0 delete (invalid image id)
(2, 1, 0) # 1 image, marker file, 0 delete
# incorrect target image
])
@patch("host.hypervisor.esx."
"image_manager.EsxImageManager._delete_single_image")
def test_image_sweeper(self,
image_id_index,
target_image_id_index,
deleted_count,
delete_single_image):
delete_single_image.side_effect = \
self.patched_delete_single_image
self.image_sweeper.image_sweep_rate = 60000
image_id = self.image_ids[image_id_index]
target_image_id = self.image_ids[target_image_id_index]
self.image_sweeper.set_target_images([target_image_id])
good_dir = os.path.join(self.test_dir,
"images",
"im",
image_id)
deleted_list = self.image_manager.\
_delete_unused_images(self.image_sweeper, good_dir)
assert_that(len(deleted_list) is deleted_count)
assert_that(self.delete_count is deleted_count)
@patch("host.hypervisor.esx."
"image_manager.EsxImageManager._delete_single_image")
def test_image_sweeper_bad_root(self, delete_single_image):
delete_single_image.side_effect = \
self.patched_delete_single_image
self.image_sweeper.image_sweep_rate = 60000
self.image_sweeper.set_target_images(["image_id_5"])
bad_dir = os.path.join(self.test_dir,
"images",
"im",
"im.vmdk")
dictionary = self.image_manager.\
_delete_unused_images(self.image_sweeper, bad_dir)
assert_that(len(dictionary) is 0)
assert_that(self.delete_count is 0)
def patched_delete_single_image(self, image_sweeper,
pathname, image_id):
self.delete_count += 1
return True
class ImageSweeperTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
IMAGE_SWEEP_RATE = 12
TIMEOUT = 6000
# This test uses two threads, the main thread is used to check
# the state inside the state machine at different stages. The
# second thread, executes the steps in sequence and calls out
# the main method:
# delete_unused()
# These methods have been patched to synchronize with the main
# thread.
def setUp(self):
self.image_manager = MagicMock()
self.image_manager.delete_unused.side_effect = self.fake_delete_unused_images
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.synchronizer = TestSynchronizer()
self.timeout = self.TIMEOUT
def tearDown(self):
self.image_sweeper.stop()
@patch(
"host.hypervisor.image_sweeper.DatastoreImageSweeperTaskRunner._delete_unused_images"
)
def test_lifecycle(self, delete_unused_images):
assert_that(
self.image_sweeper.get_state() is DatastoreImageSweeper.State.IDLE)
delete_unused_images.side_effect = self.fake_delete_unused_images
self.image_sweeper.start(self.timeout, list(), self.IMAGE_SWEEP_RATE)
self.image_sweeper.wait_for_task_end()
assert_that(
self.image_sweeper.get_state() is DatastoreImageSweeper.State.IDLE)
delete_unused_images.assert_called_once_with(self.image_sweeper,
"/vmfs/volumes/DS01")
deleted_images, _ = self.image_sweeper.get_deleted_images()
assert_that(len(deleted_images) is 3)
def fake_delete_unused_images(self, image_sweeper, datastore_root):
assert_that(image_sweeper.datastore_id is self.DATASTORE_ID)
assert_that(self.image_sweeper.get_state() is
DatastoreImageSweeper.State.IMAGE_SWEEP)
assert_that(image_sweeper.image_sweep_rate is self.IMAGE_SWEEP_RATE)
deleted_images = list()
deleted_images.append("image_id_1")
deleted_images.append("image_id_2")
deleted_images.append("image_id_3")
return deleted_images
def wait_for_runner(self):
self.synchronizer.wait_for_runner()
def signal_runner(self):
self.synchronizer.signal_runner()
def wait_for_main(self):
self.synchronizer.wait_for_main()
def signal_main(self):
self.synchronizer.signal_main()
class ImageSweeperTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
IMAGE_SWEEP_RATE = 12
TIMEOUT = 6000
# This test uses two threads, the main thread is used to check
# the state inside the state machine at different stages. The
# second thread, executes the steps in sequence and calls out
# the main method:
# delete_unused()
# These methods have been patched to synchronize with the main
# thread.
def setUp(self):
self.image_manager = MagicMock()
self.image_manager.delete_unused.side_effect = self.fake_delete_unused_images
self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID)
self.synchronizer = TestSynchronizer()
self.timeout = self.TIMEOUT
def tearDown(self):
self.image_sweeper.stop()
@patch("host.hypervisor.image_sweeper.DatastoreImageSweeperTaskRunner._delete_unused_images")
def test_lifecycle(self, delete_unused_images):
assert_that(self.image_sweeper.get_state() is DatastoreImageSweeper.State.IDLE)
delete_unused_images.side_effect = self.fake_delete_unused_images
self.image_sweeper.start(self.timeout, list(), self.IMAGE_SWEEP_RATE)
self.image_sweeper.wait_for_task_end()
assert_that(self.image_sweeper.get_state() is DatastoreImageSweeper.State.IDLE)
delete_unused_images.assert_called_once_with(self.image_sweeper, "/vmfs/volumes/DS01")
deleted_images, _ = self.image_sweeper.get_deleted_images()
assert_that(len(deleted_images) is 3)
def fake_delete_unused_images(self, image_sweeper, datastore_root):
assert_that(image_sweeper.datastore_id is self.DATASTORE_ID)
assert_that(self.image_sweeper.get_state() is DatastoreImageSweeper.State.IMAGE_SWEEP)
assert_that(image_sweeper.image_sweep_rate is self.IMAGE_SWEEP_RATE)
deleted_images = list()
deleted_images.append("image_id_1")
deleted_images.append("image_id_2")
deleted_images.append("image_id_3")
return deleted_images
def wait_for_runner(self):
self.synchronizer.wait_for_runner()
def signal_runner(self):
self.synchronizer.signal_runner()
def wait_for_main(self):
self.synchronizer.wait_for_main()
def signal_main(self):
self.synchronizer.signal_main()