def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.vim_client = MagicMock()
self.vim_client.delete_file.side_effect = self.patched_delete
self.image_manager = ImageManager(self.vim_client, MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID)
self.deleted = False
self.marker_unlinked = False
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = str(uuid.uuid4())
self.image_id_1 = image_id_1
self.image_id_2 = image_id_2
self.image_id_3 = image_id_3
self.image_id_4 = image_id_4
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
self.marker_file_content_time = 0
self.timestamp_file_mod_time = 0
# Create a good image vmdk under image_id_1, also create a valid image marker file and a valid timestamp file
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
marker_filename = os.path.join(dir1, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_2, create timestamp but no image marker file,
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir2, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
# Create a good image vmdk under image_id_3, create image_marker file but no timestamp file
vmdk_filename = image_id_3 + ".vmdk"
open(os.path.join(dir3, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir3, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_4, create image_marker file, renamed timestamp file
# but no timestamp file
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir4, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "image_")
self.dir0 = dir0
# Image dir with correct timestamp file
image_id_1 = str(uuid.uuid4())
dir1 = self.create_dir(image_id_1)
open(os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
# Image dir without the correct timestamp file
image_id_2 = str(uuid.uuid4())
dir2 = self.create_dir(image_id_2)
# Image dir with correct timestamp file
# and with tombstone file
image_id_3 = str(uuid.uuid4())
dir3 = self.create_dir(image_id_3)
open(os.path.join(dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
self.image_ids = ["", image_id_1, image_id_2, image_id_3]
self.image_dirs = ["", dir1, dir2, dir3]
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
self.host_client = Hypervisor.create_host_client(
errback=lambda: suicide())
self.host_client.connect_local()
atexit.register(lambda client: client.disconnect(), self.host_client)
self.datastore_manager = DatastoreManager(
self, agent_config.datastores, agent_config.image_datastores)
# datastore manager needs to update the cache when there is a change.
self.host_client.add_update_listener(self.datastore_manager)
self.vm_manager = VmManager(self.host_client, self.datastore_manager)
self.disk_manager = DiskManager(self.host_client,
self.datastore_manager)
self.image_manager = ImageManager(self.host_client,
self.datastore_manager)
self.network_manager = NetworkManager(self.host_client)
self.system = System(self.host_client)
options = PlacementOption(agent_config.memory_overcommit,
agent_config.cpu_overcommit,
agent_config.image_datastores)
self.placement_manager = PlacementManager(self, options)
self.image_monitor = ImageMonitor(self.datastore_manager,
self.image_manager, self.vm_manager)
self.image_manager.monitor_for_cleanup()
self.image_transferer = NfcImageTransferer(self.host_client)
atexit.register(self.image_manager.cleanup)
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.vim_client = MagicMock()
self.vim_client.delete_file.side_effect = self.patched_delete
self.image_manager = ImageManager(self.vim_client, MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID)
self.deleted = False
self.marker_unlinked = False
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = str(uuid.uuid4())
self.image_id_1 = image_id_1
self.image_id_2 = image_id_2
self.image_id_3 = image_id_3
self.image_id_4 = image_id_4
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
self.marker_file_content_time = 0
self.timestamp_file_mod_time = 0
# Create a good image vmdk under image_id_1, also create a valid image marker file and a valid timestamp file
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
marker_filename = os.path.join(dir1, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_2, create timestamp but no image marker file,
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir2, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
# Create a good image vmdk under image_id_3, create image_marker file but no timestamp file
vmdk_filename = image_id_3 + ".vmdk"
open(os.path.join(dir3, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir3, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_4, create image_marker file, renamed timestamp file
# but no timestamp file
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir4, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "image_")
self.dir0 = dir0
# Image dir with correct timestamp file
image_id_1 = str(uuid.uuid4())
dir1 = self.create_dir(image_id_1)
open(os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
# Image dir without the correct timestamp file
image_id_2 = str(uuid.uuid4())
dir2 = self.create_dir(image_id_2)
# Image dir with correct timestamp file
# and with tombstone file
image_id_3 = str(uuid.uuid4())
dir3 = self.create_dir(image_id_3)
open(os.path.join(dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
self.image_ids = ["", image_id_1, image_id_2, image_id_3]
self.image_dirs = ["", dir1, dir2, dir3]
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
self.host_client = Hypervisor.create_host_client(errback=lambda: suicide())
self.host_client.connect_local()
atexit.register(lambda client: client.disconnect(), self.host_client)
self.datastore_manager = DatastoreManager(
self, agent_config.datastores, agent_config.image_datastores)
# datastore manager needs to update the cache when there is a change.
self.host_client.add_update_listener(self.datastore_manager)
self.vm_manager = VmManager(self.host_client, self.datastore_manager)
self.disk_manager = DiskManager(self.host_client, self.datastore_manager)
self.image_manager = ImageManager(self.host_client, self.datastore_manager)
self.network_manager = NetworkManager(self.host_client)
self.system = System(self.host_client)
options = PlacementOption(agent_config.memory_overcommit,
agent_config.cpu_overcommit,
agent_config.image_datastores)
self.placement_manager = PlacementManager(self, options)
self.image_monitor = ImageMonitor(self.datastore_manager,
self.image_manager,
self.vm_manager)
self.image_manager.monitor_for_cleanup()
self.image_transferer = NfcImageTransferer(self.host_client)
atexit.register(self.image_manager.cleanup)
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_root,
_uuid):
""" Test that stray images are found and deleted by the reaper """
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_ds_dir = os.path.join(tmpdir, ds.id)
os.mkdir(tmp_ds_dir)
tmp_image_dir = os.path.join(
tmp_ds_dir,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image"))
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_root(datastore):
return os.path.join(tmpdir, datastore)
_os_datastore_root.side_effect = _fake_os_datastore_root
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = ImageManager(self.vim_client, ds_manager)
if not _allow_grace_period:
image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0
time.sleep(0.1)
image_manager.reap_tmp_images()
if _allow_grace_period:
# verify stray image is not deleted due to grace period
self.assertTrue(os.path.exists(path))
else:
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
def setUp(self):
# Create VM manager
self.vim_client = VimClient(auto_sync=False)
self.vim_client._content = MagicMock()
self.vim_client.wait_for_task = MagicMock()
self.vm_manager = VmManager(self.vim_client, MagicMock())
services.register(ServiceName.AGENT_CONFIG, MagicMock())
# Set up test files
self.base_dir = os.path.dirname(__file__)
self.test_dir = os.path.join(self.base_dir, "../test_files")
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.image_scanner = DatastoreImageScanner(self.image_manager, self.vm_manager, self.DATASTORE_ID)
self.write_count = 0
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.delete_count = 0
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = "invalid_image_id"
self.image_ids = ["*", image_id_1, image_id_2, image_id_3, image_id_4]
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
# Create a vmdk under "im", since the image_id is not a valid uuid it should be skipped
open(os.path.join(self.test_dir, "image_im.vmdk"), 'w').close()
# Create a good image vmdk under image_id_1 but no image marker file, this should not be deleted
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
# Create a good image vmdk under image_id_2, also create an unused image marker file,
# image_id_2 should be deleted
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
open(os.path.join(dir2, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a marker file under dir3 but no vmdk file. It should be deleted as well
open(os.path.join(dir3, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a vmdk under an invalid image directory, also create a marker file. Since the image_id
# is not valid it should not be deleted
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
open(os.path.join(dir4, self.IMAGE_MARKER_FILENAME), 'w').close()
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_scanner = DatastoreImageScanner(self.image_manager,
self.vm_manager,
self.DATASTORE_ID)
self.write_count = 0
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = "invalid_image_id"
self.image_ids = ["*", image_id_1, image_id_2, image_id_3, image_id_4]
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
# Create a vmdk under "im", since the image_id is not a valid uuid it should be skipped
open(os.path.join(self.test_dir, "image_im.vmdk"), 'w').close()
# Create a good image vmdk under image_id_1, the name of the vmdk matches the directory
# that contains it so this is a valid image to remove
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
# Create a good image vmdk under image_id_2, also create an unused image marker file,
# image_id_2 should also be included in the list of images to remove
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
open(
os.path.join(dir2,
self.image_manager.UNUSED_IMAGE_MARKER_FILE_NAME),
'w').close()
# Don't create anything under directory dir3. it should still mark the image as deletable
# Create a vmdk under an invalid image directory, since the image id is not valid it
# should not mark it for deletion
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
def test_periodic_reaper(self, mock_reap):
""" Test that the we invoke the image reaper periodically """
image_manager = ImageManager(self.vim_client, self.ds_manager)
image_manager.monitor_for_cleanup(reap_interval=0.1)
self.assertFalse(image_manager._image_reaper is None)
retry = 0
while mock_reap.call_count < 2 and retry < 10:
time.sleep(0.1)
retry += 1
image_manager.cleanup()
assert_that(mock_reap.call_count, greater_than(1))
assert_that(retry, is_not(10), "reaper cleanup not called repeatedly")
class ImageSweeperTouchTimestampTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
BASE_TEMP_DIR = "image_sweeper"
IMAGE_TIMESTAMP_FILENAME = ImageManager.IMAGE_TIMESTAMP_FILE_NAME
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "image_")
self.dir0 = dir0
# Image dir with correct timestamp file
image_id_1 = str(uuid.uuid4())
dir1 = self.create_dir(image_id_1)
open(os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
# Image dir without the correct timestamp file
image_id_2 = str(uuid.uuid4())
dir2 = self.create_dir(image_id_2)
# Image dir with correct timestamp file
# and with tombstone file
image_id_3 = str(uuid.uuid4())
dir3 = self.create_dir(image_id_3)
open(os.path.join(dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
self.image_ids = ["", image_id_1, image_id_2, image_id_3]
self.image_dirs = ["", dir1, dir2, dir3]
def tearDown(self):
shutil.rmtree(self.test_dir, True)
@parameterized.expand([
# timestamp_exists
(True, ),
(False, )
])
# The os_vmdk_path method is defined in vm_config.py but it is imported in esx/image_manager.py, that is
# the instance we need to patch
@patch("host.image.image_manager.os_vmdk_path")
def test_touch_timestamp_file(self, timestamp_exists, os_vmdk_path):
if not timestamp_exists:
image_index = 2
exception_class = type(OSError())
else:
image_index = 1
exception_class = None
image_id = self.image_ids[image_index]
image_dir = self.image_dirs[image_index]
os_vmdk_path.side_effect = self.patched_os_vmdk_path
timestamp_filename_path = os.path.join(image_dir, self.IMAGE_TIMESTAMP_FILENAME)
pre_mod_time = 0
if timestamp_exists:
# save mod time on the image timestamp file
pre_mod_time = os.path.getmtime(timestamp_filename_path)
try:
time.sleep(1)
self.image_manager.touch_image_timestamp(self.DATASTORE_ID, image_id)
assert_that(exception_class is None)
# check new timestamp
post_mod_time = os.path.getmtime(timestamp_filename_path)
assert_that((post_mod_time > pre_mod_time) is True)
except Exception as ex:
assert_that((type(ex) == exception_class) is True)
def patched_os_vmdk_path(self, datastore, disk_id, folder):
folder = self.dir0
ret = os_vmdk_path(datastore, disk_id, folder)
return ret
def create_dir(self, image_id):
dirname = vm_config.compond_path_join(self.dir0, image_id)
os.makedirs(dirname)
return dirname
class Hypervisor(object):
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
self.host_client = Hypervisor.create_host_client(errback=lambda: suicide())
self.host_client.connect_local()
atexit.register(lambda client: client.disconnect(), self.host_client)
self.datastore_manager = DatastoreManager(
self, agent_config.datastores, agent_config.image_datastores)
# datastore manager needs to update the cache when there is a change.
self.host_client.add_update_listener(self.datastore_manager)
self.vm_manager = VmManager(self.host_client, self.datastore_manager)
self.disk_manager = DiskManager(self.host_client, self.datastore_manager)
self.image_manager = ImageManager(self.host_client, self.datastore_manager)
self.network_manager = NetworkManager(self.host_client)
self.system = System(self.host_client)
options = PlacementOption(agent_config.memory_overcommit,
agent_config.cpu_overcommit,
agent_config.image_datastores)
self.placement_manager = PlacementManager(self, options)
self.image_monitor = ImageMonitor(self.datastore_manager,
self.image_manager,
self.vm_manager)
self.image_manager.monitor_for_cleanup()
self.image_transferer = NfcImageTransferer(self.host_client)
atexit.register(self.image_manager.cleanup)
@staticmethod
def create_host_client(auto_sync=True, errback=None):
try:
# check whether attache is installed. If not, find_module will throw ImportError.
from host.hypervisor.esx.attache_client import AttacheClient
return AttacheClient(auto_sync, errback)
except ImportError:
from host.hypervisor.esx.vim_client import VimClient
return VimClient(auto_sync, errback)
def check_image(self, image_id, datastore_id):
return self.image_manager.check_image(
image_id, self.datastore_manager.datastore_name(datastore_id)
)
def get_resources(self):
return self.vm_manager.get_resources()
def get_vm_resource(self, vm_id):
vm = self.vm_manager.get_resource(vm_id)
resource = Resource(vm=vm)
return resource
def acquire_vim_ticket(self):
return self.host_client.get_vim_ticket()
def add_update_listener(self, listener):
self.host_client.add_update_listener(listener)
def remove_update_listener(self, listener):
self.host_client.remove_update_listener(listener)
def transfer_image(self, source_image_id, source_datastore,
destination_image_id, destination_datastore,
host, port):
return self.image_transferer.send_image_to_host(
source_image_id, source_datastore,
destination_image_id, destination_datastore, host, port)
@property
def memory_overcommit(self):
return self.placement_manager.memory_overcommit
def set_memory_overcommit(self, value):
self.placement_manager.memory_overcommit = value
@property
def cpu_overcommit(self):
return self.placement_manager.cpu_overcommit
def set_cpu_overcommit(self, value):
self.placement_manager.cpu_overcommit = value
class ImageSweeperDeleteImageTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
BASE_TEMP_DIR = "delete_image"
IMAGE_MARKER_FILENAME = ImageManager.UNUSED_IMAGE_MARKER_FILE_NAME
IMAGE_TIMESTAMP_FILENAME = ImageManager.IMAGE_TIMESTAMP_FILE_NAME
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.vim_client = MagicMock()
self.vim_client.delete_file.side_effect = self.patched_delete
self.image_manager = ImageManager(self.vim_client, MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID)
self.deleted = False
self.marker_unlinked = False
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = str(uuid.uuid4())
self.image_id_1 = image_id_1
self.image_id_2 = image_id_2
self.image_id_3 = image_id_3
self.image_id_4 = image_id_4
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
self.marker_file_content_time = 0
self.timestamp_file_mod_time = 0
# Create a good image vmdk under image_id_1, also create a valid image marker file and a valid timestamp file
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
marker_filename = os.path.join(dir1, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_2, create timestamp but no image marker file,
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir2, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
# Create a good image vmdk under image_id_3, create image_marker file but no timestamp file
vmdk_filename = image_id_3 + ".vmdk"
open(os.path.join(dir3, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir3, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_4, create image_marker file, renamed timestamp file
# but no timestamp file
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir4, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
def tearDown(self):
shutil.rmtree(self.test_dir, True)
# The following test plays with the content of marker file (a timestamp) and the mod time of
# the image timestamp file before and after rename. It should delete the image only if
# marker - grace period > timestamp AND timestamp == timestamp after rename
# All the other cases should point to the fact that the image has been used after the image
# scan started. To avoid problems due to non synchronized clocks on different hosts
# a grace period of 10 minutes is applied on the timestamp from the marker file
@parameterized.expand([
# marker time, mod time, deleted
(1061, 1000, True),
(1060, 1000, False),
(1000, 1000, False),
(1000, 1001, False)
])
@patch("host.image.image_manager.ImageManager._image_directory")
@patch("host.image.image_manager.ImageManager._read_marker_file")
@patch("host.image.image_manager.ImageManager._get_datastore_type")
@patch("host.image.image_manager.ImageManager._get_mod_time")
@patch("os.unlink")
def test_delete_image(self, marker_file_content_time, timestamp_file_mod_time, deleted,
unlink, get_mod_time, get_datastore_type, read_marker_file, image_directory):
self.marker_file_content_time = marker_file_content_time
self.timestamp_file_mod_time = timestamp_file_mod_time
marker_unlinked = not deleted
read_marker_file.side_effect = self.patched_read_marker_file
get_datastore_type.side_effect = self.patched_get_datastore_type
get_mod_time.side_effect = self.patched_get_mod_time
unlink.side_effect = self.patched_unlink
image_directory.return_value = os.path.join(self.test_dir, "image_" + self.image_id_1)
ret = self.image_manager.delete_image(self.DATASTORE_ID, self.image_id_1, 60)
assert_that(deleted is ret)
assert_that(deleted is self.deleted)
assert_that(marker_unlinked is self.marker_unlinked)
@patch("host.image.image_manager.ImageManager._image_directory")
@patch("host.image.image_manager.ImageManager._get_datastore_type")
@patch("host.image.image_manager.ImageManager._get_mod_time")
def test_delete_image_no_marker_file(self, get_mod_time, get_datastore_type, image_directory):
get_datastore_type.side_effect = self.patched_get_datastore_type
get_mod_time.side_effect = self.patched_get_mod_time
image_directory.return_value = os.path.join(self.test_dir, "image_" + self.image_id_2)
deleted = self.image_manager.delete_image(self.DATASTORE_ID, self.image_id_2, 60)
assert_that(deleted is False)
assert_that(self.deleted is False)
@patch("host.image.image_manager.ImageManager._image_directory")
@patch("host.image.image_manager.ImageManager._read_marker_file")
@patch("host.image.image_manager.ImageManager._get_datastore_type")
def test_delete_image_no_timestamp_file(self, get_datastore_type, read_marker_file, image_directory):
read_marker_file.side_effect = self.patched_read_marker_file
get_datastore_type.side_effect = self.patched_get_datastore_type
self.marker_file_content_time = 1000
image_directory.return_value = os.path.join(self.test_dir, "image_" + self.image_id_3)
deleted = self.image_manager.delete_image(self.DATASTORE_ID, self.image_id_3, 60)
assert_that(deleted is True)
assert_that(self.deleted is True)
def patched_read_marker_file(self, filename):
if not os.path.exists(filename):
raise OSError
return self.marker_file_content_time
def patched_get_datastore_type(self, datastore_id):
return DatastoreType.EXT3
def patched_get_mod_time(self, filename):
try:
os.path.getmtime(filename)
except OSError as ex:
if ex.errno == errno.ENOENT:
return False, 0
else:
raise ex
# fix mod_time
mod_time = self.timestamp_file_mod_time
return True, mod_time
def patched_unlink(self, target):
self.marker_unlinked = True
def patched_delete(self, target):
self.deleted = True
class Hypervisor(object):
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
self.host_client = Hypervisor.create_host_client(
errback=lambda: suicide())
self.host_client.connect_local()
atexit.register(lambda client: client.disconnect(), self.host_client)
self.datastore_manager = DatastoreManager(
self, agent_config.datastores, agent_config.image_datastores)
# datastore manager needs to update the cache when there is a change.
self.host_client.add_update_listener(self.datastore_manager)
self.vm_manager = VmManager(self.host_client, self.datastore_manager)
self.disk_manager = DiskManager(self.host_client,
self.datastore_manager)
self.image_manager = ImageManager(self.host_client,
self.datastore_manager)
self.network_manager = NetworkManager(self.host_client)
self.system = System(self.host_client)
options = PlacementOption(agent_config.memory_overcommit,
agent_config.cpu_overcommit,
agent_config.image_datastores)
self.placement_manager = PlacementManager(self, options)
self.image_monitor = ImageMonitor(self.datastore_manager,
self.image_manager, self.vm_manager)
self.image_manager.monitor_for_cleanup()
self.image_transferer = NfcImageTransferer(self.host_client)
atexit.register(self.image_manager.cleanup)
@staticmethod
def create_host_client(auto_sync=True, errback=None):
try:
# check whether attache is installed. If not, find_module will throw ImportError.
from host.hypervisor.esx.attache_client import AttacheClient
return AttacheClient(auto_sync, errback)
except ImportError:
from host.hypervisor.esx.vim_client import VimClient
return VimClient(auto_sync, errback)
def check_image(self, image_id, datastore_id):
return self.image_manager.check_image(
image_id, self.datastore_manager.datastore_name(datastore_id))
def get_resources(self):
return self.vm_manager.get_resources()
def get_vm_resource(self, vm_id):
vm = self.vm_manager.get_resource(vm_id)
resource = Resource(vm=vm)
return resource
def acquire_vim_ticket(self):
return self.host_client.get_vim_ticket()
def add_update_listener(self, listener):
self.host_client.add_update_listener(listener)
def remove_update_listener(self, listener):
self.host_client.remove_update_listener(listener)
def transfer_image(self, source_image_id, source_datastore,
destination_image_id, destination_datastore, host,
port):
return self.image_transferer.send_image_to_host(
source_image_id, source_datastore, destination_image_id,
destination_datastore, host, port)
@property
def memory_overcommit(self):
return self.placement_manager.memory_overcommit
def set_memory_overcommit(self, value):
self.placement_manager.memory_overcommit = value
@property
def cpu_overcommit(self):
return self.placement_manager.cpu_overcommit
def set_cpu_overcommit(self, value):
self.placement_manager.cpu_overcommit = value
class ImageSweeperTouchTimestampTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
BASE_TEMP_DIR = "image_sweeper"
IMAGE_TIMESTAMP_FILENAME = ImageManager.IMAGE_TIMESTAMP_FILE_NAME
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "image_")
self.dir0 = dir0
# Image dir with correct timestamp file
image_id_1 = str(uuid.uuid4())
dir1 = self.create_dir(image_id_1)
open(os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
# Image dir without the correct timestamp file
image_id_2 = str(uuid.uuid4())
dir2 = self.create_dir(image_id_2)
# Image dir with correct timestamp file
# and with tombstone file
image_id_3 = str(uuid.uuid4())
dir3 = self.create_dir(image_id_3)
open(os.path.join(dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
self.image_ids = ["", image_id_1, image_id_2, image_id_3]
self.image_dirs = ["", dir1, dir2, dir3]
def tearDown(self):
shutil.rmtree(self.test_dir, True)
@parameterized.expand([
# timestamp_exists
(True, ),
(False, )
])
# The os_vmdk_path method is defined in vm_config.py but it is imported in esx/image_manager.py, that is
# the instance we need to patch
@patch("host.image.image_manager.os_vmdk_path")
def test_touch_timestamp_file(self, timestamp_exists, os_vmdk_path):
if not timestamp_exists:
image_index = 2
exception_class = type(OSError())
else:
image_index = 1
exception_class = None
image_id = self.image_ids[image_index]
image_dir = self.image_dirs[image_index]
os_vmdk_path.side_effect = self.patched_os_vmdk_path
timestamp_filename_path = os.path.join(image_dir, self.IMAGE_TIMESTAMP_FILENAME)
pre_mod_time = 0
if timestamp_exists:
# save mod time on the image timestamp file
pre_mod_time = os.path.getmtime(timestamp_filename_path)
try:
time.sleep(1)
self.image_manager.touch_image_timestamp(self.DATASTORE_ID, image_id)
assert_that(exception_class is None)
# check new timestamp
post_mod_time = os.path.getmtime(timestamp_filename_path)
assert_that((post_mod_time > pre_mod_time) is True)
except Exception as ex:
assert_that((type(ex) == exception_class) is True)
def patched_os_vmdk_path(self, datastore, disk_id, folder):
folder = self.dir0
ret = os_vmdk_path(datastore, disk_id, folder)
return ret
def create_dir(self, image_id):
dirname = vm_config.compond_path_join(self.dir0, image_id)
os.makedirs(dirname)
return dirname
def setUp(self):
self.vim_client = VimClient(auto_sync=False)
self.vim_client._content = MagicMock()
self.ds_manager = MagicMock()
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(self.vim_client, self.ds_manager)
class TestImageManager(unittest.TestCase):
"""Image Manager tests."""
# We can use even more unit test coverage of the image manager here
def setUp(self):
self.vim_client = VimClient(auto_sync=False)
self.vim_client._content = MagicMock()
self.ds_manager = MagicMock()
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = ImageManager(self.vim_client, self.ds_manager)
@patch("host.image.image_manager.ImageManager.reap_tmp_images")
def test_periodic_reaper(self, mock_reap):
""" Test that the we invoke the image reaper periodically """
image_manager = ImageManager(self.vim_client, self.ds_manager)
image_manager.monitor_for_cleanup(reap_interval=0.1)
self.assertFalse(image_manager._image_reaper is None)
retry = 0
while mock_reap.call_count < 2 and retry < 10:
time.sleep(0.1)
retry += 1
image_manager.cleanup()
assert_that(mock_reap.call_count, greater_than(1))
assert_that(retry, is_not(10), "reaper cleanup not called repeatedly")
@parameterized.expand([(True, ), (False, )])
@patch("uuid.uuid4", return_value="fake_id")
@patch("host.hypervisor.esx.path_util.os_datastore_root")
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_root,
_uuid):
""" Test that stray images are found and deleted by the reaper """
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_ds_dir = os.path.join(tmpdir, ds.id)
os.mkdir(tmp_ds_dir)
tmp_image_dir = os.path.join(
tmp_ds_dir,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image"))
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_root(datastore):
return os.path.join(tmpdir, datastore)
_os_datastore_root.side_effect = _fake_os_datastore_root
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = ImageManager(self.vim_client, ds_manager)
if not _allow_grace_period:
image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0
time.sleep(0.1)
image_manager.reap_tmp_images()
if _allow_grace_period:
# verify stray image is not deleted due to grace period
self.assertTrue(os.path.exists(path))
else:
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
@patch("pysdk.task.WaitForTask")
@patch("uuid.uuid4", return_value="fake_id")
@patch("os.path.exists")
@patch("shutil.copy")
@patch.object(VimClient, "move_file")
@patch.object(VimClient, "copy_disk")
@patch.object(ImageManager,
"_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(ImageManager, "_check_image_repair", return_value=False)
@patch.object(ImageManager, "check_and_validate_image", return_value=False)
@patch.object(ImageManager, "_create_image_timestamp_file")
@patch("host.image.image_manager.FileBackedLock")
def test_copy_image(self, _flock, _create_image_timestamp, check_image,
_check_image_repair, _get_ds_type, _copy_disk, _mv_dir,
_copy, _exists, _uuid, _wait_for_task):
_exists.side_effect = (
True, # tmp_dir exists
True, # dest image vmdk missing
True) # source meta file present
self.image_manager.copy_image("ds1", "foo", "ds2", "bar")
os_path_prefix1 = '/vmfs/volumes/ds1'
os_path_prefix2 = '/vmfs/volumes/ds2'
assert_that(_copy.call_count, equal_to(1))
_copy.assert_has_calls([
call(
'%s/image_foo/foo.%s' % (os_path_prefix1, METADATA_FILE_EXT),
'/vmfs/volumes/ds2/tmp_image_fake_id/bar.%s' %
METADATA_FILE_EXT),
])
ds_path_prefix1 = '[] ' + os_path_prefix1
expected_tmp_disk_ds_path = '%s/tmp_image_fake_id/bar.vmdk' % (
os_path_prefix2)
_copy_disk.assert_called_once_with(
'%s/image_foo/foo.vmdk' % ds_path_prefix1,
expected_tmp_disk_ds_path)
_mv_dir.assert_called_once_with('/vmfs/volumes/ds2/tmp_image_fake_id',
'%s/image_bar' % os_path_prefix2)
_create_image_timestamp.assert_called_once_with(
"/vmfs/volumes/ds2/tmp_image_fake_id")
@patch("pysdk.task.WaitForTask")
@patch("uuid.uuid4", return_value="fake_id")
@patch("os.path.exists")
@patch("os.makedirs")
@patch("shutil.copy")
@patch.object(VimClient, "copy_disk")
@patch.object(ImageManager,
"_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(ImageManager, "check_image", return_value=False)
@patch.object(ImageManager, "_create_image_timestamp_file")
def test_create_tmp_image(self, _create_image_timestamp, check_image,
_get_ds_type, _copy_disk, _copy, _makedirs,
_exists, _uuid, _wait_for_task):
# Common case is the same as the one covered by test_copy_image.
# Check that things work when the src metadata file doesn't exist.
_exists.side_effect = (False, False, True)
ds_path_prefix1 = '[] /vmfs/volumes/ds1'
expected_tmp_disk_ds_path = "/vmfs/volumes/ds2/tmp_image_fake_id/bar.vmdk"
self.image_manager._copy_to_tmp_image("ds1", "foo", "ds2", "bar")
# Verify that we don't copy the metadata file.
self.assertFalse(_copy.called)
# Verify that we copy the disk correctly
_copy_disk.assert_called_once_with(
'%s/image_foo/foo.vmdk' % ds_path_prefix1,
expected_tmp_disk_ds_path)
# check that we return an IO error if the copy of metadata fails.
_copy.side_effect = IOError
_exists.side_effect = (True, True)
_copy_disk.reset_mock()
self.assertRaises(IOError, self.image_manager._copy_to_tmp_image,
"ds1", "foo", "ds2", "bar")
self.assertFalse(_copy_disk.called)
_create_image_timestamp.assert_called_once_with(
"/vmfs/volumes/ds2/tmp_image_fake_id")
@patch.object(VimClient, "delete_file")
@patch("os.path.exists", return_value=True)
@patch("os.makedirs")
@patch("shutil.rmtree")
@patch("shutil.move")
@patch.object(ImageManager,
"_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(ImageManager, "_check_image_repair", return_value=True)
@patch("host.image.image_manager.FileBackedLock")
@raises(DiskAlreadyExistException)
def test_move_image(self, _flock, check_image, _get_ds_type, _mv_dir,
_rmtree, _makedirs, _exists, _delete_file):
# Common case is covered in test_copy_image.
# check that if destination image directory exists we don't call move
# and just bail after removing the tmp dir
_rmtree.reset_mock()
_mv_dir.reset_mock()
expected_tmp_disk_folder = '/vmfs/volumes/ds2/tmp_images/bar'
expected_rm_calls = [call(expected_tmp_disk_folder)]
self.image_manager._move_image("foo", "ds1", expected_tmp_disk_folder)
self.assertEqual(expected_rm_calls, _rmtree.call_args_list)
_makedirs.assert_called_once_with('/vmfs/volumes/ds1/images/fo')
_flock.assert_called_once_with('/vmfs/volumes/ds1/image_foo',
DatastoreType.EXT3, 3)
_delete_file.assert_called_once_with('/vmfs/volumes/ds1/image_foo')
@parameterized.expand([(True, ), (False, )])
@patch("os.path.exists")
@patch.object(ImageManager,
"_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(ImageManager, "_create_image_timestamp_file")
@patch("host.image.image_manager.FileBackedLock")
def test_validate_existing_image(self, create, _flock,
_create_timestamp_file, _get_ds_type,
_path_exists):
self._create_image_timestamp_file = create
_path_exists.side_effect = self._local_os_path_exists
_disk_folder = '/vmfs/volumes/ds1/image_foo'
self.image_manager._check_image_repair("foo", "ds1")
if create:
_create_timestamp_file.assert_called_once_with(_disk_folder)
else:
assert not _create_timestamp_file.called
def _local_os_path_exists(self, pathname):
if not self._create_image_timestamp_file:
return True
if pathname.endswith(ImageManager.IMAGE_TIMESTAMP_FILE_NAME):
return False
else:
return True
def test_image_path(self):
image_path = "/vmfs/volumes/ds/image_ttylinux/ttylinux.vmdk"
ds = self.image_manager.get_datastore_id_from_path(image_path)
image = self.image_manager.get_image_id_from_path(image_path)
self.assertEqual(ds, "ds")
self.assertEqual(image, "ttylinux")
@patch.object(ImageManager, "_get_datastore_type")
def test_create_image(self, _get_ds_type):
image_id = "image_id"
datastore_id = "ds1"
_get_ds_type.side_effect = (DatastoreType.LOCAL_VMFS,
DatastoreType.VSAN)
tmp_image_path = self.image_manager.create_image(
image_id, datastore_id)
prefix = "[] /vmfs/volumes/%s/tmp_image_" % datastore_id
self.assertTrue(tmp_image_path.startswith(prefix))
tmp_image_path = self.image_manager.create_image(
image_id, datastore_id)
prefix = "[] /vmfs/volumes/%s/image_%s/tmp_image_" % (datastore_id,
image_id)
self.assertTrue(tmp_image_path.startswith(prefix))
@patch.object(ImageManager, "_move_image")
@patch.object(ImageManager, "_create_image_timestamp_file")
@patch("os.path.exists")
def test_finalize_image(self, _exists, _create_timestamp, move_image):
# Happy path verify move is called with the right args.
_exists.side_effect = ([True])
self.image_manager.finalize_image("ds1", "[] /vmfs/volumes/ds1/foo",
"img_1")
move_image.assert_called_once_with('img_1', 'ds1',
'/vmfs/volumes/ds1/foo')
_create_timestamp.assert_called_once_with(
"/vmfs/volumes/ds1/image_img_1")
@patch.object(ImageManager, "finalize_image")
@patch.object(VimClient, "copy_disk")
@patch("os.path.exists", return_value=True)
def test_create_image_with_vm_disk(self, _exists, _copy_disk,
_create_image):
vm_disk_path = "/vmfs/volumes/dsname/vms/ab/cd.vmdk"
self.image_manager.create_image_with_vm_disk(
"ds1", "[] /vmfs/volumes/ds1/foo", "img_1", vm_disk_path)
# Verify that we copy the disk correctly
expected_tmp_disk_ds_path = "/vmfs/volumes/ds1/foo/img_1.vmdk"
_copy_disk.assert_called_once_with(vm_disk_path,
expected_tmp_disk_ds_path)
_create_image.assert_called_once_with("ds1",
"[] /vmfs/volumes/ds1/foo",
"img_1")
@patch("shutil.rmtree")
@patch("os.path.exists")
def test_delete_tmp_dir(self, _exists, _rmtree):
self.image_manager.delete_tmp_dir("ds1", "foo")
_exists.assert_called_once("/vmfs/volumes/ds1/foo")
_rmtree.assert_called_once("/vmfs/volumes/ds1/foo")
_exists.reset_mock()
_exists.return_value = False
_rmtree.reset_mock()
self.assertRaises(DirectoryNotFound, self.image_manager.delete_tmp_dir,
"ds1", "foo")
_exists.assert_called_once("/vmfs/volumes/ds1/foo")
self.assertFalse(_rmtree.called)
def test_image_size(self):
self.ds_manager.image_datastores.return_value = ["ds1", "ds2"]
with patch("host.image.image_manager.os_vmdk_flat_path"
"") as image_path:
tmpdir = file_util.mkdtemp(delete=True)
image_path.return_value = tmpdir
size = self.image_manager.image_size("image_id")
self.assertTrue(size > 0)
def test_image_size_not_exist(self):
self.ds_manager.image_datastores.return_value = ["ds1", "ds2"]
self.assertRaises(NoSuchResourceException,
self.image_manager.image_size, "image_id")
class ImageSweeperDeleteImageTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
BASE_TEMP_DIR = "delete_image"
IMAGE_MARKER_FILENAME = ImageManager.UNUSED_IMAGE_MARKER_FILE_NAME
IMAGE_TIMESTAMP_FILENAME = ImageManager.IMAGE_TIMESTAMP_FILE_NAME
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.vim_client = MagicMock()
self.vim_client.delete_file.side_effect = self.patched_delete
self.image_manager = ImageManager(self.vim_client, MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID)
self.deleted = False
self.marker_unlinked = False
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = str(uuid.uuid4())
self.image_id_1 = image_id_1
self.image_id_2 = image_id_2
self.image_id_3 = image_id_3
self.image_id_4 = image_id_4
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
self.marker_file_content_time = 0
self.timestamp_file_mod_time = 0
# Create a good image vmdk under image_id_1, also create a valid image marker file and a valid timestamp file
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
marker_filename = os.path.join(dir1, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_2, create timestamp but no image marker file,
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir2, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
# Create a good image vmdk under image_id_3, create image_marker file but no timestamp file
vmdk_filename = image_id_3 + ".vmdk"
open(os.path.join(dir3, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir3, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_4, create image_marker file, renamed timestamp file
# but no timestamp file
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir4, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
def tearDown(self):
shutil.rmtree(self.test_dir, True)
# The following test plays with the content of marker file (a timestamp) and the mod time of
# the image timestamp file before and after rename. It should delete the image only if
# marker - grace period > timestamp AND timestamp == timestamp after rename
# All the other cases should point to the fact that the image has been used after the image
# scan started. To avoid problems due to non synchronized clocks on different hosts
# a grace period of 10 minutes is applied on the timestamp from the marker file
@parameterized.expand([
# marker time, mod time, deleted
(1061, 1000, True),
(1060, 1000, False),
(1000, 1000, False),
(1000, 1001, False)
])
@patch("host.image.image_manager.ImageManager._image_directory")
@patch("host.image.image_manager.ImageManager._read_marker_file")
@patch("host.image.image_manager.ImageManager._get_datastore_type")
@patch("host.image.image_manager.ImageManager._get_mod_time")
@patch("os.unlink")
def test_delete_image(self, marker_file_content_time, timestamp_file_mod_time, deleted,
unlink, get_mod_time, get_datastore_type, read_marker_file, image_directory):
self.marker_file_content_time = marker_file_content_time
self.timestamp_file_mod_time = timestamp_file_mod_time
marker_unlinked = not deleted
read_marker_file.side_effect = self.patched_read_marker_file
get_datastore_type.side_effect = self.patched_get_datastore_type
get_mod_time.side_effect = self.patched_get_mod_time
unlink.side_effect = self.patched_unlink
image_directory.return_value = os.path.join(self.test_dir, "image_" + self.image_id_1)
ret = self.image_manager.delete_image(self.DATASTORE_ID, self.image_id_1, 60)
assert_that(deleted is ret)
assert_that(deleted is self.deleted)
assert_that(marker_unlinked is self.marker_unlinked)
@patch("host.image.image_manager.ImageManager._image_directory")
@patch("host.image.image_manager.ImageManager._get_datastore_type")
@patch("host.image.image_manager.ImageManager._get_mod_time")
def test_delete_image_no_marker_file(self, get_mod_time, get_datastore_type, image_directory):
get_datastore_type.side_effect = self.patched_get_datastore_type
get_mod_time.side_effect = self.patched_get_mod_time
image_directory.return_value = os.path.join(self.test_dir, "image_" + self.image_id_2)
deleted = self.image_manager.delete_image(self.DATASTORE_ID, self.image_id_2, 60)
assert_that(deleted is False)
assert_that(self.deleted is False)
@patch("host.image.image_manager.ImageManager._image_directory")
@patch("host.image.image_manager.ImageManager._read_marker_file")
@patch("host.image.image_manager.ImageManager._get_datastore_type")
def test_delete_image_no_timestamp_file(self, get_datastore_type, read_marker_file, image_directory):
read_marker_file.side_effect = self.patched_read_marker_file
get_datastore_type.side_effect = self.patched_get_datastore_type
self.marker_file_content_time = 1000
image_directory.return_value = os.path.join(self.test_dir, "image_" + self.image_id_3)
deleted = self.image_manager.delete_image(self.DATASTORE_ID, self.image_id_3, 60)
assert_that(deleted is True)
assert_that(self.deleted is True)
def patched_read_marker_file(self, filename):
if not os.path.exists(filename):
raise OSError
return self.marker_file_content_time
def patched_get_datastore_type(self, datastore_id):
return DatastoreType.EXT3
def patched_get_mod_time(self, filename):
try:
os.path.getmtime(filename)
except OSError as ex:
if ex.errno == errno.ENOENT:
return False, 0
else:
raise ex
# fix mod_time
mod_time = self.timestamp_file_mod_time
return True, mod_time
def patched_unlink(self, target):
self.marker_unlinked = True
def patched_delete(self, target):
self.deleted = True
