def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "image_")
self.dir0 = dir0
# Image dir with correct timestamp file
image_id_1 = str(uuid.uuid4())
dir1 = self.create_dir(image_id_1)
open(os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
# Image dir without the correct timestamp file
image_id_2 = str(uuid.uuid4())
dir2 = self.create_dir(image_id_2)
# Image dir with correct timestamp file
# and with tombstone file
image_id_3 = str(uuid.uuid4())
dir3 = self.create_dir(image_id_3)
open(os.path.join(
dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
self.image_ids = ["", image_id_1, image_id_2, image_id_3]
self.image_dirs = ["", dir1, dir2, dir3]
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout,
errback=lambda: suicide())
atexit.register(lambda client: client.disconnect(), self.vim_client)
self._uuid = self.vim_client.host_uuid
self.set_memory_overcommit(agent_config.memory_overcommit)
self.datastore_manager = EsxDatastoreManager(
self, agent_config.datastores, agent_config.image_datastores)
# datastore manager needs to update the cache when there is a change.
self.vim_client.add_update_listener(self.datastore_manager)
self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager)
self.disk_manager = EsxDiskManager(self.vim_client,
self.datastore_manager)
self.image_manager = EsxImageManager(self.vim_client,
self.datastore_manager)
self.network_manager = EsxNetworkManager(self.vim_client,
agent_config.networks)
self.system = EsxSystem(self.vim_client)
self.image_manager.monitor_for_cleanup()
self.image_transferer = HttpNfcTransferer(
self.vim_client, self.datastore_manager.image_datastores())
atexit.register(self.image_manager.cleanup)
def test_reap_tmp_images(self, _os_datastore_path, _uuid):
""" Test that stray images are found and deleted by the reaper """
def _fake_ds_folder(datastore, folder):
return "%s__%s" % (datastore, folder)
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_images_folder = _fake_ds_folder(ds.id, TMP_IMAGE_FOLDER_NAME)
tmp_images_dir = os.path.join(tmpdir, tmp_images_folder)
tmp_image_dir = os.path.join(tmp_images_dir, "stray_image")
os.mkdir(tmp_images_dir)
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_path(datastore, folder):
return os.path.join(tmpdir, _fake_ds_folder(datastore, folder))
_os_datastore_path.side_effect = _fake_os_datastore_path
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = EsxImageManager(self.vim_client, ds_manager)
image_manager.reap_tmp_images()
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
def test_reap_tmp_images(self, _os_datastore_path, _uuid):
""" Test that stray images are found and deleted by the reaper """
def _fake_ds_folder(datastore, folder):
return "%s__%s" % (datastore, folder)
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_images_folder = _fake_ds_folder(ds.id, TMP_IMAGE_FOLDER_NAME)
tmp_images_dir = os.path.join(tmpdir, tmp_images_folder)
tmp_image_dir = os.path.join(tmp_images_dir, "stray_image")
os.mkdir(tmp_images_dir)
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_path(datastore, folder):
return os.path.join(tmpdir, _fake_ds_folder(datastore, folder))
_os_datastore_path.side_effect = _fake_os_datastore_path
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = EsxImageManager(self.vim_client, ds_manager)
image_manager.reap_tmp_images()
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
def test_periodic_reaper(self, mock_reap):
""" Test that the we invoke the image reaper periodically """
image_manager = EsxImageManager(self.vim_client, self.ds_manager)
image_manager.monitor_for_cleanup(reap_interval=0.1)
self.assertFalse(image_manager._image_reaper is None)
retry = 0
while mock_reap.call_count < 2 and retry < 10:
time.sleep(0.1)
retry += 1
image_manager.cleanup()
assert_that(mock_reap.call_count, greater_than(1))
assert_that(retry, is_not(10), "reaper cleanup not called repeatedly")
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
errback = lambda: suicide()
self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout,
errback=errback)
atexit.register(lambda client: client.disconnect(), self.vim_client)
self._uuid = self.vim_client.host_uuid
# Enable/Disable large page support. If this host is removed
# from the deployment, large page support will need to be
# explicitly updated by the user.
disable_large_pages = agent_config.memory_overcommit > 1.0
self.vim_client.set_large_page_support(disable=disable_large_pages)
image_datastores = [ds["name"] for ds in agent_config.image_datastores]
self.datastore_manager = EsxDatastoreManager(
self, agent_config.datastores, image_datastores)
# datastore manager needs to update the cache when there is a change.
self.vim_client.add_update_listener(self.datastore_manager)
self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager)
self.disk_manager = EsxDiskManager(self.vim_client,
self.datastore_manager)
self.image_manager = EsxImageManager(self.vim_client,
self.datastore_manager)
self.network_manager = EsxNetworkManager(self.vim_client,
agent_config.networks)
self.system = EsxSystem(self.vim_client)
self.image_manager.monitor_for_cleanup()
atexit.register(self.image_manager.cleanup)
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
errback = lambda: suicide()
self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout,
errback=errback)
atexit.register(lambda client: client.disconnect(), self.vim_client)
self._uuid = self.vim_client.host_uuid
self.set_memory_overcommit(agent_config.memory_overcommit)
image_datastores = [ds["name"] for ds in agent_config.image_datastores]
self.datastore_manager = EsxDatastoreManager(
self, agent_config.datastores, agent_config.image_datastores)
# datastore manager needs to update the cache when there is a change.
self.vim_client.add_update_listener(self.datastore_manager)
self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager)
self.disk_manager = EsxDiskManager(self.vim_client,
self.datastore_manager)
self.image_manager = EsxImageManager(self.vim_client,
self.datastore_manager)
self.network_manager = EsxNetworkManager(self.vim_client,
agent_config.networks)
self.system = EsxSystem(self.vim_client)
self.image_manager.monitor_for_cleanup()
self.image_transferer = HttpNfcTransferer(self.vim_client,
image_datastores)
atexit.register(self.image_manager.cleanup)
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_path,
_uuid):
""" Test that stray images are found and deleted by the reaper """
def _fake_ds_folder(datastore, folder):
return "%s/%s" % (datastore, folder)
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_ds_dir = os.path.join(tmpdir, ds.id)
os.mkdir(tmp_ds_dir)
tmp_image_dir = os.path.join(
tmp_ds_dir,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image"))
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_path(datastore, folder):
return os.path.join(tmpdir, _fake_ds_folder(datastore, folder))
_os_datastore_path.side_effect = _fake_os_datastore_path
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = EsxImageManager(self.vim_client, ds_manager)
if not _allow_grace_period:
image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0
time.sleep(0.1)
image_manager.reap_tmp_images()
if _allow_grace_period:
# verify stray image is not deleted due to grace period
self.assertTrue(os.path.exists(path))
else:
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_path,
_uuid):
""" Test that stray images are found and deleted by the reaper """
def _fake_ds_folder(datastore, folder):
return "%s/%s" % (datastore, folder)
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_ds_dir = os.path.join(tmpdir, ds.id)
os.mkdir(tmp_ds_dir)
tmp_image_dir = os.path.join(tmp_ds_dir, compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image"))
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_path(datastore, folder):
return os.path.join(tmpdir, _fake_ds_folder(datastore, folder))
_os_datastore_path.side_effect = _fake_os_datastore_path
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = EsxImageManager(self.vim_client, ds_manager)
if not _allow_grace_period:
image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0
time.sleep(0.1)
image_manager.reap_tmp_images()
if _allow_grace_period:
# verify stray image is not deleted due to grace period
self.assertTrue(os.path.exists(path))
else:
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, "images/im")
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = "invalid_image_id"
self.image_ids = ["", image_id_1, image_id_2, image_id_3, image_id_4]
dir1 = os.path.join(dir0, image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(dir0, image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(dir0, image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(dir0, image_id_4)
os.makedirs(dir4)
# Create a vmdk under "im", since the image_id is
# not a valid uuid it should be skipped
open(os.path.join(dir0, "im.vmdk"), 'w').close()
# Create a good image vmdk under image_id_1 but
# no image marker file, this should not be deleted
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
# Create a good image vmdk under image_id_2, also create
# an unused image marker file, image_id_2 should be
# deleted
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
open(os.path.join(dir2, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a marker file under dir3 but no
# vmdk file. It should be deleted as well
open(os.path.join(dir3, self.IMAGE_MARKER_FILENAME), 'w').close()
# Create a vmdk under an invalid image directory,
# also create a marker file. Since the image_id
# is not valid it should not be deleted
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
open(os.path.join(dir4, self.IMAGE_MARKER_FILENAME), 'w').close()
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_scanner = DatastoreImageScanner(self.image_manager,
self.vm_manager,
self.DATASTORE_ID)
self.image_scanner._task_runner = MagicMock()
self.image_scanner._task_runner.is_stopped.return_value = False
self.write_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, "images/im")
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = "invalid_image_id"
self.image_ids = ["", image_id_1, image_id_2, image_id_3, image_id_4]
dir1 = os.path.join(dir0, image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(dir0, image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(dir0, image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(dir0, image_id_4)
os.makedirs(dir4)
# Create a vmdk under "im", since the image_id is
# not a valid uuid it should be skipped
open(os.path.join(dir0, "im.vmdk"), 'w').close()
# Create a good image vmdk under image_id_1, the name
# of the vmdk matches the directory that contains it
# so this is a valid image to remove
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
# Create a good image vmdk under image_id_2, also create
# an unused image marker file, image_id_2 should also be
# included in the list of images to remove
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
open(os.path.join(dir2, self.image_manager.IMAGE_MARKER_FILE_NAME),
'w').close()
# Don't create anything under directory dir3
# it should still mark the image as deletable
# Create a vmdk under an invalid image directory, since
# the image id is not valid it should not mark it
# for deletion
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
errback = lambda: suicide()
self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout,
errback=errback)
atexit.register(lambda client: client.disconnect(), self.vim_client)
self._uuid = self.vim_client.host_uuid
# Enable/Disable large page support. If this host is removed
# from the deployment, large page support will need to be
# explicitly updated by the user.
disable_large_pages = agent_config.memory_overcommit > 1.0
self.vim_client.set_large_page_support(disable=disable_large_pages)
image_datastores = [ds["name"] for ds in agent_config.image_datastores]
self.datastore_manager = EsxDatastoreManager(self,
agent_config.datastores,
image_datastores)
# datastore manager needs to update the cache when there is a change.
self.vim_client.add_update_listener(self.datastore_manager)
self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager)
self.disk_manager = EsxDiskManager(self.vim_client,
self.datastore_manager)
self.image_manager = EsxImageManager(self.vim_client,
self.datastore_manager)
self.network_manager = EsxNetworkManager(self.vim_client,
agent_config.networks)
self.system = EsxSystem(self.vim_client)
self.image_manager.monitor_for_cleanup()
self.image_transferer = HttpNfcTransferer(self.vim_client,
image_datastores)
atexit.register(self.image_manager.cleanup)
def setUp(self, connect, update, creds):
# Create VM manager
creds.return_value = ["username", "password"]
self.vim_client = VimClient(auto_sync=False)
self.vim_client.wait_for_task = MagicMock()
self.patcher = patch("host.hypervisor.esx.vm_config.GetEnv")
self.patcher.start()
self.vm_manager = EsxVmManager(self.vim_client, MagicMock())
services.register(ServiceName.AGENT_CONFIG, MagicMock())
# Set up test files
self.base_dir = os.path.dirname(__file__)
self.test_dir = os.path.join(self.base_dir, "../../test_files")
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.image_scanner = DatastoreImageScanner(self.image_manager,
self.vm_manager,
self.DATASTORE_ID)
self.write_count = 0
def test_periodic_reaper(self, mock_reap):
""" Test that the we invoke the image reaper periodically """
image_manager = EsxImageManager(self.vim_client, self.ds_manager)
image_manager.monitor_for_cleanup(reap_interval=0.1)
self.assertFalse(image_manager._image_reaper is None)
retry = 0
while mock_reap.call_count < 2 and retry < 10:
time.sleep(0.1)
retry += 1
image_manager.cleanup()
assert_that(mock_reap.call_count, greater_than(1))
assert_that(retry, is_not(10), "reaper cleanup not called repeatedly")
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "images/im")
self.dir0 = dir0
# Image dir with correct timestamp file
image_id_1 = str(uuid.uuid4())
dir1 = self.create_dir(image_id_1)
open(os.path.join(
dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
# Image dir without the correct timestamp file
image_id_2 = str(uuid.uuid4())
dir2 = self.create_dir(image_id_2)
# Image dir with correct timestamp file
# and with tombstone file
image_id_3 = str(uuid.uuid4())
dir3 = self.create_dir(image_id_3)
open(os.path.join(
dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
open(os.path.join(
dir3, self.IMAGE_TOMBSTONE_FILENAME), 'w').close()
self.image_ids = ["", image_id_1, image_id_2, image_id_3]
self.image_dirs = ["", dir1, dir2, dir3]
class ImageSweeperDeleteSingleImageTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
BASE_TEMP_DIR = "delete_single_image"
IMAGE_MARKER_FILENAME = EsxImageManager.IMAGE_MARKER_FILE_NAME
IMAGE_TIMESTAMP_FILENAME = EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME
IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX = EsxImageManager.IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
self.gc_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.deleted = False
self.marker_unlinked = False
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = str(uuid.uuid4())
self.image_id_1 = image_id_1
self.image_id_2 = image_id_2
self.image_id_3 = image_id_3
self.image_id_4 = image_id_4
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
self.marker_file_content_time = 0
self.timestamp_file_mod_time = 0
self.renamed_timestamp_file_mod_time = 0
# Create a good image vmdk under image_id_1,
# also create a valid image marker file
# and a valid timestamp file
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
marker_filename = os.path.join(dir1, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_2,
# create timestamp but no image marker file,
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir2, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
# Create a good image vmdk under image_id_3,
# create image_marker file but no timestamp
# and no renamed timestamp file
vmdk_filename = image_id_3 + ".vmdk"
open(os.path.join(dir3, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir3, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_4,
# create image_marker file, renamed timestamp file
# but no timestamp file
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir4, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
timestamp_filename = os.path.join(dir4, self.IMAGE_TIMESTAMP_FILENAME)
renamed_timestamp_filename = timestamp_filename + self.IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX
open(renamed_timestamp_filename, 'w').close()
def tearDown(self):
shutil.rmtree(self.test_dir, True)
shutil.rmtree(self.gc_dir, True)
# The following test plays with the content of
# marker file (a timestamp) and the mod time of
# the image timestamp file before and after rename.
# It should delete the image only if
# marker - grace period > timestamp AND
# timestamp == timestamp after rename
# All the other cases should point to the fact
# that the image has been used after the image
# scan started. To avoid problems due to
# non synchronized clocks on different hosts
# a grace period of 10 minutes is applied on
# the timestamp from the marker file
@parameterized.expand([
# marker time, mod time, mod time after rename, deleted
(1061, 1000, 1000, True),
(1060, 1000, 1000, False),
(1000, 1000, 1000, False),
(1000, 1001, 1001, False),
(2000, 1000, 2010, False)
])
@patch("host.hypervisor.esx.image_manager.EsxImageManager._read_marker_file")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._get_datastore_type")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._get_mod_time")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rename")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_unlink")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rm_rf")
def test_delete_single_image(
self,
marker_file_content_time,
timestamp_file_mod_time,
renamed_timestamp_file_mod_time,
deleted,
rm_rf,
unlink,
rename,
get_mod_time,
get_datastore_type,
read_marker_file):
self.marker_file_content_time = marker_file_content_time
self.timestamp_file_mod_time = timestamp_file_mod_time
self.renamed_timestamp_file_mod_time = renamed_timestamp_file_mod_time
marker_unlinked = not deleted
read_marker_file.side_effect = self.patched_read_marker_file
get_datastore_type.side_effect = self.patched_get_datastore_type
get_mod_time.side_effect = self.patched_get_mod_time
rename.side_effect = self.patched_rename
unlink.side_effect = self.patched_unlink
rm_rf.side_effect = self.patched_rm_rf
good_dir = os.path.join(self.test_dir, "image_" + self.image_id_1)
ret = self.image_manager._delete_single_image(self.image_sweeper, good_dir, self.image_id_1)
assert_that(deleted is ret)
assert_that(deleted is self.deleted)
assert_that(marker_unlinked is self.marker_unlinked)
@patch("host.hypervisor.esx.image_manager.EsxImageManager._get_datastore_type")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._get_mod_time")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rename")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rm_rf")
def test_delete_single_image_no_marker_file(
self,
rm_rf,
rename,
get_mod_time,
get_datastore_type):
get_datastore_type.side_effect = self.patched_get_datastore_type
get_mod_time.side_effect = self.patched_get_mod_time
rename.side_effect = self.patched_rename
rm_rf.side_effect = self.patched_rm_rf
good_dir = os.path.join(self.test_dir, "image_" + self.image_id_2)
deleted = self.image_manager._delete_single_image(self.image_sweeper, good_dir, self.image_id_2)
assert_that(deleted is False)
assert_that(self.deleted is False)
@patch("host.hypervisor.esx.image_manager.EsxImageManager._read_marker_file")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._get_datastore_type")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rename")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rm_rf")
def test_delete_single_image_no_timestamp_files(
self,
rm_rf,
rename,
get_datastore_type,
read_marker_file):
read_marker_file.side_effect = self.patched_read_marker_file
get_datastore_type.side_effect = self.patched_get_datastore_type
rename.side_effect = self.patched_rename
rm_rf.side_effect = self.patched_rm_rf
self.marker_file_content_time = 1000
good_dir = os.path.join(self.test_dir, "image_" + self.image_id_3)
deleted = self.image_manager._delete_single_image(self.image_sweeper, good_dir, self.image_id_3)
assert_that(deleted is True)
assert_that(self.deleted is True)
@parameterized.expand([
# marker time, mod time, mod time after rename, deleted
(1061, 1000, True),
(1060, 1000, False),
(1000, 1000, False),
(1000, 1001, False)
])
@patch("host.hypervisor.esx.image_manager.EsxImageManager._read_marker_file")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._get_datastore_type")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._get_mod_time")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rename")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_unlink")
@patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rm_rf")
def test_delete_single_image_no_timestamp_file(
self,
marker_file_content_time,
renamed_timestamp_file_mod_time,
deleted,
rm_rf,
unlink,
rename,
get_mod_time,
get_datastore_type,
read_marker_file):
self.marker_file_content_time = marker_file_content_time
self.renamed_timestamp_file_mod_time = renamed_timestamp_file_mod_time
marker_unlinked = not deleted
read_marker_file.side_effect = self.patched_read_marker_file
get_datastore_type.side_effect = self.patched_get_datastore_type
get_mod_time.side_effect = self.patched_get_mod_time
rename.side_effect = self.patched_rename
unlink.side_effect = self.patched_unlink
rm_rf.side_effect = self.patched_rm_rf
good_dir = os.path.join(self.test_dir, "image_" + self.image_id_4)
ret = self.image_manager._delete_single_image(self.image_sweeper, good_dir, self.image_id_4)
assert_that(deleted is ret)
assert_that(deleted is self.deleted)
assert_that(marker_unlinked is self.marker_unlinked)
def patched_read_marker_file(self, filename):
if not os.path.exists(filename):
raise OSError
return self.marker_file_content_time
def patched_get_datastore_type(self, datastore_id):
return DatastoreType.EXT3
def patched_get_mod_time(self, filename):
try:
os.path.getmtime(filename)
except OSError as ex:
if ex.errno == errno.ENOENT:
return False, 0
else:
raise ex
# fix mod_time
if filename.endswith(
self.IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX):
mod_time = self.renamed_timestamp_file_mod_time
else:
mod_time = self.timestamp_file_mod_time
return True, mod_time
def patched_rename(self, source, destination):
if source.endswith(self.IMAGE_TIMESTAMP_FILENAME):
shutil.move(source, destination)
else:
shutil.move(source, self.gc_dir)
def patched_unlink(self, target):
self.marker_unlinked = True
def patched_rm_rf(self, target):
self.deleted = True
def setUp(self, connect, update, creds):
creds.return_value = ["username", "password"]
self.vim_client = VimClient(auto_sync=False)
self.ds_manager = MagicMock()
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(self.vim_client, self.ds_manager)
def setUp(self, connect, update, creds):
creds.return_value = ["username", "password"]
self.vim_client = VimClient(auto_sync=False)
self.ds_manager = MagicMock()
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(self.vim_client, self.ds_manager)
class TestEsxImageManager(unittest.TestCase):
"""Image Manager tests."""
# We can use even more unit test coverage of the image manager here
@patch.object(VimClient, "acquire_credentials")
@patch.object(VimClient, "update_cache")
@patch("pysdk.connect.Connect")
def setUp(self, connect, update, creds):
creds.return_value = ["username", "password"]
self.vim_client = VimClient(auto_sync=False)
self.ds_manager = MagicMock()
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(self.vim_client, self.ds_manager)
def tearDown(self):
self.vim_client.disconnect(wait=True)
@patch("os.path.isdir", return_value=False)
@patch("os.makedirs", side_effect=OSError)
def test_make_image_dir(self, _makedirs, _isdir):
self.assertRaises(
OSError, self.image_manager._make_image_dir, "ds", "fake_iid")
_isdir.assert_called_once_with("/vmfs/volumes/ds/images/fa/fake_iid")
self.assertEqual(
_makedirs.call_count, EsxImageManager.NUM_MAKEDIRS_ATTEMPTS)
for i in range(0, EsxImageManager.NUM_MAKEDIRS_ATTEMPTS):
self.assertEqual(_makedirs.call_args_list[i][0],
("/vmfs/volumes/ds/images/fa/fake_iid",))
@patch(
"host.hypervisor.esx.image_manager.EsxImageManager.reap_tmp_images")
def test_periodic_reaper(self, mock_reap):
""" Test that the we invoke the image reaper periodically """
image_manager = EsxImageManager(self.vim_client, self.ds_manager)
image_manager.monitor_for_cleanup(reap_interval=0.1)
self.assertFalse(image_manager._image_reaper is None)
retry = 0
while mock_reap.call_count < 2 and retry < 10:
time.sleep(0.1)
retry += 1
image_manager.cleanup()
assert_that(mock_reap.call_count, greater_than(1))
assert_that(retry, is_not(10), "reaper cleanup not called repeatedly")
@patch("uuid.uuid4", return_value="fake_id")
@patch("host.hypervisor.esx.vm_config.os_datastore_path")
def test_reap_tmp_images(self, _os_datastore_path, _uuid):
""" Test that stray images are found and deleted by the reaper """
def _fake_ds_folder(datastore, folder):
return "%s__%s" % (datastore, folder)
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_images_folder = _fake_ds_folder(ds.id, TMP_IMAGE_FOLDER_NAME)
tmp_images_dir = os.path.join(tmpdir, tmp_images_folder)
tmp_image_dir = os.path.join(tmp_images_dir, "stray_image")
os.mkdir(tmp_images_dir)
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_path(datastore, folder):
return os.path.join(tmpdir, _fake_ds_folder(datastore, folder))
_os_datastore_path.side_effect = _fake_os_datastore_path
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = EsxImageManager(self.vim_client, ds_manager)
image_manager.reap_tmp_images()
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
@patch("os.path.isdir")
@patch("os.makedirs")
def test_vmdk_mkdir_eexist(self, _makedirs, _isdir):
eexist = OSError()
eexist.errno = errno.EEXIST
_makedirs.side_effect = eexist
_isdir.side_effect = (False, # dest image dir missing
True) # dest image dir is created
self.image_manager._make_image_dir("ds", "fake_iid")
_isdir.assert_called("/vmfs/volumes/ds/images/fa/fake_iid")
@patch("pysdk.task.WaitForTask")
@patch("uuid.uuid4", return_value="fake_id")
@patch("os.path.exists")
@patch("os.makedirs")
@patch("shutil.copy")
@patch("shutil.rmtree")
@patch("shutil.move")
@patch.object(EsxImageManager, "_manage_disk")
@patch.object(EsxImageManager, "_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(EsxImageManager, "_check_image_repair", return_value=False)
@patch.object(EsxImageManager,
"check_and_validate_image", return_value=False)
@patch.object(EsxImageManager, "_create_image_timestamp_file")
@patch("host.hypervisor.esx.image_manager.FileBackedLock")
def test_copy_image(self, _flock, _create_image_timestamp,
check_image, _check_image_repair,
_get_ds_type, _manage_disk,
_mv_dir, _rmtree, _copy, _makedirs, _exists,
_uuid, _wait_for_task):
_exists.side_effect = (True, # dest image vmdk missing
True) # source meta file present
self.image_manager.copy_image("ds1", "foo", "ds2", "bar")
os_path_prefix1 = '/vmfs/volumes/ds1/images'
os_path_prefix2 = '/vmfs/volumes/ds2/images'
os_tmp_path_prefix = '/vmfs/volumes/ds2/tmp_images'
_copy.assert_called_once_with(
'%s/fo/foo/foo.%s' % (os_path_prefix1, METADATA_FILE_EXT),
'/vmfs/volumes/ds2/tmp_images/fake_id')
ds_path_prefix1 = '[] ' + os_path_prefix1
ds_tmp_path_prefix = '[] ' + os_tmp_path_prefix
expected_tmp_disk_ds_path = '%s/fake_id/%s.vmdk' % (ds_tmp_path_prefix,
'bar')
_vd_spec = _manage_disk.call_args_list[0][1]['destSpec']
self.assertEqual("thin", _vd_spec.diskType)
self.assertEqual("lsiLogic", _vd_spec.adapterType)
copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName='%s/fo/foo/foo.vmdk' % ds_path_prefix1,
destName=expected_tmp_disk_ds_path,
destSpec=_vd_spec)
expected_vim_calls = [copy_call]
self.assertEqual(expected_vim_calls, _manage_disk.call_args_list)
_mv_dir.assert_called_once_with('/vmfs/volumes/ds2/tmp_images/fake_id',
'%s/ba/bar' % os_path_prefix2)
_create_image_timestamp.assert_called_once_with(
"/vmfs/volumes/ds2/tmp_images/fake_id")
@patch("pysdk.task.WaitForTask")
@patch("uuid.uuid4", return_value="fake_id")
@patch("os.path.exists")
@patch("os.makedirs")
@patch("shutil.copy")
@patch.object(EsxImageManager, "_manage_disk")
@patch.object(EsxImageManager, "_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(EsxImageManager, "check_image", return_value=False)
@patch.object(EsxImageManager, "_create_image_timestamp_file")
@patch("host.hypervisor.esx.image_manager.FileBackedLock")
def test_create_tmp_image(self, _flock, _create_image_timestamp,
check_image, _get_ds_type,
_manage_disk, _copy, _makedirs, _exists,
_uuid, _wait_for_task):
# Common case is the same as the one covered by test_copy_image.
# Check that things work when the src metadata file doesn't exist.
_exists.side_effect = (False, True)
ds_path_prefix1 = '[] /vmfs/volumes/ds1/images'
expected_tmp_disk_ds_path = \
"[] /vmfs/volumes/ds2/tmp_images/fake_id/bar.vmdk"
self.image_manager._create_tmp_image("ds1", "foo", "ds2", "bar")
_flock.assert_called_once_with("/vmfs/volumes/ds2/tmp_images/fake_id",
DatastoreType.EXT3)
# Verify that we don't copy the metadata file.
self.assertFalse(_copy.called)
# Verify that we copy the disk correctly
_vd_spec = _manage_disk.call_args_list[0][1]['destSpec']
self.assertEqual("thin", _vd_spec.diskType)
self.assertEqual("lsiLogic", _vd_spec.adapterType)
copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName='%s/fo/foo/foo.vmdk' % ds_path_prefix1,
destName=expected_tmp_disk_ds_path,
destSpec=_vd_spec)
expected_vim_calls = [copy_call]
self.assertEqual(expected_vim_calls, _manage_disk.call_args_list)
# check that we return an IO error if the copy of metadata fails.
_copy.side_effect = IOError
_exists.side_effect = (True, True)
_manage_disk.reset_mock()
_flock.reset_mock()
self.assertRaises(IOError, self.image_manager._create_tmp_image,
"ds1", "foo", "ds2", "bar")
self.assertFalse(_manage_disk.called)
_flock.assert_called_once_with("/vmfs/volumes/ds2/tmp_images/fake_id",
DatastoreType.EXT3)
_create_image_timestamp.assert_called_once_with(
"/vmfs/volumes/ds2/tmp_images/fake_id")
@patch("os.makedirs")
@patch("shutil.rmtree")
@patch("shutil.move")
@patch.object(EsxImageManager, "_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(EsxImageManager, "_check_image_repair", return_value=True)
@patch("host.hypervisor.esx.image_manager.FileBackedLock")
@raises(DiskAlreadyExistException)
def test_move_image(self, _flock, check_image, _get_ds_type, _mv_dir,
_rmtree, _makedirs):
# Common case is covered in test_copy_image.
# check that if destination image directory exists we don't call move
# and just bail after removing the tmp dir
_rmtree.reset_mock()
_mv_dir.reset_mock()
expected_tmp_disk_folder = '/vmfs/volumes/ds2/tmp_images/bar'
expected_rm_calls = [call(expected_tmp_disk_folder)]
self.image_manager._move_image("foo", "ds1", expected_tmp_disk_folder)
self.assertEqual(expected_rm_calls, _rmtree.call_args_list)
_makedirs.assert_called_once_with('/vmfs/volumes/ds1/images/fo')
_flock.assert_called_once_with('/vmfs/volumes/ds1/images/fo/foo',
DatastoreType.EXT3, 3)
@parameterized.expand([
(True, ),
(False, )
])
@patch("os.path.exists")
@patch.object(EsxImageManager, "_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(EsxImageManager, "_create_image_timestamp_file")
@patch.object(EsxImageManager, "_delete_renamed_image_timestamp_file")
@patch("host.hypervisor.esx.image_manager.FileBackedLock")
def test_validate_existing_image(self,
create,
_flock,
_delete_renamed_timestamp_file,
_create_timestamp_file,
_get_ds_type,
_path_exists):
self._create_image_timestamp_file = create
_path_exists.side_effect = self._local_os_path_exists
_disk_folder = '/vmfs/volumes/ds1/images/fo/foo'
self.image_manager._check_image_repair("foo", "ds1")
if create:
_create_timestamp_file.assert_called_once_with(_disk_folder)
_delete_renamed_timestamp_file.assert_called_once()
else:
assert not _create_timestamp_file.called
assert not _delete_renamed_timestamp_file.called
def _local_os_path_exists(self, pathname):
if not self._create_image_timestamp_file:
return True
if pathname.endswith(EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME):
return False
else:
return True
@patch.object(EsxImageManager, "_clean_gc_dir")
@patch.object(EsxImageManager, "_gc_image_dir")
@patch.object(EsxImageManager, "_lock_data_disk")
@patch.object(EsxImageManager, "create_image_tombstone")
@patch.object(EsxImageManager, "check_image_dir")
def test_delete(self, check_image_dir, create_image_tombstone,
lock_data_disk, gc_image_dir, clean_gc_dir):
# Test successful delete
check_image_dir.return_value = True
self.image_manager.delete_image("ds1", "foo", 0, False)
check_image_dir.assert_called_with("foo", "ds1")
create_image_tombstone.assert_called_with("ds1", "foo")
# Test successful delete with force option
self.image_manager.delete_image("ds1", "foo", 0, True)
check_image_dir.assert_called_with("foo", "ds1")
create_image_tombstone.assert_called_with("ds1", "foo")
lock_data_disk.assert_called_with("ds1", "foo")
gc_image_dir.assert_called_with("ds1", "foo")
clean_gc_dir.assert_called()
# Test image not found
check_image_dir.return_value = False
self.assertRaises(ImageNotFoundException,
self.image_manager.delete_image,
"ds1", "foo", 0, False)
@patch("host.hypervisor.esx.image_manager.os_vmdk_path")
@patch("host.hypervisor.esx.image_manager.os_datastore_path")
def test_gc_image_dir(self, dst_path, src_path):
""" Test that we move the directory correctly to the GC location """
src_dir = file_util.mkdtemp(delete=True)
dst_dir = file_util.mkdtemp(delete=True)
src_path.return_value = os.path.join(src_dir, "test.vmdk")
dst_path.return_value = dst_dir
self.image_manager._gc_image_dir("ds1", "foo")
uuid_dir = os.path.join(dst_dir, os.listdir(dst_dir)[0])
# Verify the src directory has been moved into the garbage dir.
self.assertEqual(os.listdir(uuid_dir), [os.path.basename(src_dir)])
src_path.assert_called_once_with("ds1", "foo", IMAGE_FOLDER_NAME)
dst_path.assert_called_once_with("ds1", GC_IMAGE_FOLDER)
def test_image_path(self):
image_path = "/vmfs/volumes/ds/images/tt/ttylinux/ttylinux.vmdk"
ds = self.image_manager.get_datastore_id_from_path(image_path)
image = self.image_manager.get_image_id_from_path(image_path)
self.assertEqual(ds, "ds")
self.assertEqual(image, "ttylinux")
@patch("host.hypervisor.esx.image_manager.os_vmdk_flat_path")
@patch("host.hypervisor.esx.image_manager.os.remove")
def test_lock_data_disk(self, mock_rm, vmdk_flat_path):
""" Test acquisition of the lock on the flat file. """
vmdk_flat_path.return_value = "fake_f_name"
self.assertTrue(self.image_manager._lock_data_disk("ds1", "foo"))
vmdk_flat_path.assert_called_once_with("ds1", "foo")
mock_rm.side_effect = OSError
self.assertFalse(self.image_manager._lock_data_disk("ds1", "foo"))
@parameterized.expand([
("CLOUD", "EAGER", ImageType.CLOUD, ImageReplication.EAGER),
("MANAGEMENT", "EAGER", ImageType.MANAGEMENT, ImageReplication.EAGER),
("CLOUD", "ON_DEMAND", ImageType.CLOUD, ImageReplication.ON_DEMAND),
("MANAGEMENT", "ON_DEMAND", ImageType.MANAGEMENT,
ImageReplication.ON_DEMAND),
])
def test_image_type(self, type, replication, expected_type,
expected_replication):
self.ds_manager.image_datastores.return_value = "ds1"
with patch("host.hypervisor.esx.image_manager.os_image_manifest_path"
"") as manifest_path:
tmpdir = file_util.mkdtemp(delete=True)
tmpfile = os.path.join(tmpdir, "ds1.manifest")
manifest_path.return_value = tmpfile
with open(tmpfile, 'w+') as f:
f.write('{"imageType":"%s","imageReplication":"%s"}' % (
type, replication))
type, replication = self.image_manager.get_image_manifest(
"image_id")
self.assertEqual(type, expected_type)
self.assertEqual(replication, expected_replication)
@patch.object(EsxImageManager, "_move_image")
@patch.object(EsxImageManager, "check_image_dir", return_value=False)
@patch.object(EsxImageManager, "_create_image_timestamp_file_from_ids")
@patch("os.path.exists")
def test_create_image(self, _exists, _create_timestamp,
check_image_dir, move_image):
# Happy path verify move is called with the right args.
_exists.side_effect = ([True])
self.image_manager.create_image("ds1", "foo", "img_1")
check_image_dir.assert_called_once_with("img_1", "ds1")
move_image.assert_called_once_with('img_1', 'ds1',
'/vmfs/volumes/ds1/foo')
_create_timestamp.assert_called_once_with("ds1", "img_1")
# Verify error if tmp image doesn't exist
_exists.side_effect = ([False])
move_image.reset_mock()
self.assertRaises(ImageNotFoundException,
self.image_manager.create_image,
"ds1", "foo", "img_1")
self.assertFalse(move_image.called)
# Verify error if destination image already exists.
_exists.side_effect = ([True])
move_image.reset_mock()
check_image_dir.return_value = True
self.assertRaises(DiskAlreadyExistException,
self.image_manager.create_image,
"ds1", "foo", "img_1")
self.assertFalse(move_image.called)
@patch.object(EsxImageManager, "create_image")
@patch.object(EsxImageManager, "_manage_disk")
@patch("os.path.exists", return_value=True)
def test_create_image_with_vm_disk(self, _exists, _manage_disk,
_create_image):
vm_disk_path = "/vmfs/volumes/dsname/vms/ab/cd.vmdk"
self.image_manager.create_image_with_vm_disk(
"ds1", "foo", "img_1", vm_disk_path)
# Verify that we copy the disk correctly
expected_tmp_disk_ds_path = \
"[] /vmfs/volumes/ds1/foo/img_1.vmdk"
_vd_spec = _manage_disk.call_args_list[0][1]['destSpec']
self.assertEqual("thin", _vd_spec.diskType)
self.assertEqual("lsiLogic", _vd_spec.adapterType)
copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName='[] %s' % vm_disk_path,
destName=expected_tmp_disk_ds_path,
destSpec=_vd_spec)
expected_vim_calls = [copy_call]
self.assertEqual(expected_vim_calls, _manage_disk.call_args_list)
_create_image.assert_called_once_with("ds1", "foo", "img_1")
@patch("shutil.rmtree")
@patch("os.path.exists")
def test_delete_tmp_dir(self, _exists, _rmtree):
self.image_manager.delete_tmp_dir("ds1", "foo")
_exists.assert_called_once("/vmfs/volumes/ds1/foo")
_rmtree.assert_called_once("/vmfs/volumes/ds1/foo")
_exists.reset_mock()
_exists.return_value = False
_rmtree.reset_mock()
self.assertRaises(DirectoryNotFound,
self.image_manager.delete_tmp_dir,
"ds1", "foo")
_exists.assert_called_once("/vmfs/volumes/ds1/foo")
self.assertFalse(_rmtree.called)
class TestEsxImageManager(unittest.TestCase):
"""Image Manager tests."""
# We can use even more unit test coverage of the image manager here
@patch.object(VimClient, "acquire_credentials")
@patch.object(VimClient, "update_cache")
@patch("pysdk.connect.Connect")
def setUp(self, connect, update, creds):
creds.return_value = ["username", "password"]
self.vim_client = VimClient(auto_sync=False)
self.ds_manager = MagicMock()
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(self.vim_client, self.ds_manager)
def tearDown(self):
self.vim_client.disconnect(wait=True)
@patch("os.path.isdir", return_value=False)
@patch("os.makedirs", side_effect=OSError)
def test_make_image_dir(self, _makedirs, _isdir):
path = "/vmfs/volumes/ds/image_fake_iid"
self.assertRaises(
OSError, self.image_manager._make_image_dir, "ds", "fake_iid")
_isdir.assert_called_once_with(path)
self.assertEqual(
_makedirs.call_count, EsxImageManager.NUM_MAKEDIRS_ATTEMPTS)
for i in range(0, EsxImageManager.NUM_MAKEDIRS_ATTEMPTS):
self.assertEqual(_makedirs.call_args_list[i][0], (path,))
@patch(
"host.hypervisor.esx.image_manager.EsxImageManager.reap_tmp_images")
def test_periodic_reaper(self, mock_reap):
""" Test that the we invoke the image reaper periodically """
image_manager = EsxImageManager(self.vim_client, self.ds_manager)
image_manager.monitor_for_cleanup(reap_interval=0.1)
self.assertFalse(image_manager._image_reaper is None)
retry = 0
while mock_reap.call_count < 2 and retry < 10:
time.sleep(0.1)
retry += 1
image_manager.cleanup()
assert_that(mock_reap.call_count, greater_than(1))
assert_that(retry, is_not(10), "reaper cleanup not called repeatedly")
@parameterized.expand([
(True, ),
(False, )
])
@patch("uuid.uuid4", return_value="fake_id")
@patch("host.hypervisor.esx.vm_config.os_datastore_root")
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_root,
_uuid):
""" Test that stray images are found and deleted by the reaper """
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_ds_dir = os.path.join(tmpdir, ds.id)
os.mkdir(tmp_ds_dir)
tmp_image_dir = os.path.join(tmp_ds_dir, compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image"))
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_root(datastore):
return os.path.join(tmpdir, datastore)
_os_datastore_root.side_effect = _fake_os_datastore_root
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = EsxImageManager(self.vim_client, ds_manager)
if not _allow_grace_period:
image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0
time.sleep(0.1)
image_manager.reap_tmp_images()
if _allow_grace_period:
# verify stray image is not deleted due to grace period
self.assertTrue(os.path.exists(path))
else:
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
@patch("os.path.isdir")
@patch("os.makedirs")
def test_vmdk_mkdir_eexist(self, _makedirs, _isdir):
eexist = OSError()
eexist.errno = errno.EEXIST
_makedirs.side_effect = eexist
_isdir.side_effect = (False, # dest image dir missing
True) # dest image dir is created
self.image_manager._make_image_dir("ds", "fake_iid")
_isdir.assert_called("/vmfs/volumes/ds/image_fake_iid")
@patch("pysdk.task.WaitForTask")
@patch("uuid.uuid4", return_value="fake_id")
@patch("os.path.exists")
@patch("shutil.copy")
@patch.object(VimClient, "move_file")
@patch.object(EsxImageManager, "_manage_disk")
@patch.object(EsxImageManager, "_get_datastore_type", return_value=DatastoreType.EXT3)
@patch.object(EsxImageManager, "_check_image_repair", return_value=False)
@patch.object(EsxImageManager, "check_and_validate_image", return_value=False)
@patch.object(EsxImageManager, "_create_image_timestamp_file")
@patch("host.hypervisor.esx.image_manager.FileBackedLock")
def test_copy_image(self, _flock, _create_image_timestamp, check_image, _check_image_repair,
_get_ds_type, _manage_disk, _mv_dir, _copy, _exists, _uuid, _wait_for_task):
_exists.side_effect = (True, # tmp_dir exists
True, # dest image vmdk missing
True) # source meta file present
self.image_manager.copy_image("ds1", "foo", "ds2", "bar")
os_path_prefix1 = '/vmfs/volumes/ds1'
os_path_prefix2 = '/vmfs/volumes/ds2'
ds_tmp_path_prefix = '[] /vmfs/volumes/ds2'
assert_that(_copy.call_count, equal_to(1))
_copy.assert_has_calls([
call('%s/image_foo/foo.%s' % (os_path_prefix1, METADATA_FILE_EXT),
'/vmfs/volumes/ds2/tmp_image_fake_id/bar.%s' %
METADATA_FILE_EXT),
])
ds_path_prefix1 = '[] ' + os_path_prefix1
expected_tmp_disk_ds_path = '%s/tmp_image_fake_id/bar.vmdk' % (ds_tmp_path_prefix)
_vd_spec = _manage_disk.call_args_list[0][1]['destSpec']
self.assertEqual("thin", _vd_spec.diskType)
self.assertEqual("lsiLogic", _vd_spec.adapterType)
copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName='%s/image_foo/foo.vmdk' % ds_path_prefix1,
destName=expected_tmp_disk_ds_path,
destSpec=_vd_spec)
expected_vim_calls = [copy_call]
self.assertEqual(expected_vim_calls, _manage_disk.call_args_list)
_mv_dir.assert_called_once_with('/vmfs/volumes/ds2/tmp_image_fake_id',
'%s/image_bar' % os_path_prefix2)
_create_image_timestamp.assert_called_once_with(
"/vmfs/volumes/ds2/tmp_image_fake_id")
@patch("pysdk.task.WaitForTask")
@patch("uuid.uuid4", return_value="fake_id")
@patch("os.path.exists")
@patch("os.makedirs")
@patch("shutil.copy")
@patch.object(EsxImageManager, "_manage_disk")
@patch.object(EsxImageManager, "_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(EsxImageManager, "check_image", return_value=False)
@patch.object(EsxImageManager, "_create_image_timestamp_file")
def test_create_tmp_image(self, _create_image_timestamp, check_image, _get_ds_type,
_manage_disk, _copy, _makedirs, _exists, _uuid, _wait_for_task):
# Common case is the same as the one covered by test_copy_image.
# Check that things work when the src metadata file doesn't exist.
_exists.side_effect = (False, False, True)
ds_path_prefix1 = '[] /vmfs/volumes/ds1'
expected_tmp_disk_ds_path = \
"[] /vmfs/volumes/ds2/tmp_image_fake_id/bar.vmdk"
self.image_manager._copy_to_tmp_image("ds1", "foo", "ds2", "bar")
# Verify that we don't copy the metadata file.
self.assertFalse(_copy.called)
# Verify that we copy the disk correctly
_vd_spec = _manage_disk.call_args_list[0][1]['destSpec']
self.assertEqual("thin", _vd_spec.diskType)
self.assertEqual("lsiLogic", _vd_spec.adapterType)
copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName='%s/image_foo/foo.vmdk' % ds_path_prefix1,
destName=expected_tmp_disk_ds_path,
destSpec=_vd_spec)
expected_vim_calls = [copy_call]
self.assertEqual(expected_vim_calls, _manage_disk.call_args_list)
# check that we return an IO error if the copy of metadata fails.
_copy.side_effect = IOError
_exists.side_effect = (True, True)
_manage_disk.reset_mock()
self.assertRaises(IOError, self.image_manager._copy_to_tmp_image,
"ds1", "foo", "ds2", "bar")
self.assertFalse(_manage_disk.called)
_create_image_timestamp.assert_called_once_with(
"/vmfs/volumes/ds2/tmp_image_fake_id")
@patch.object(VimClient, "delete_file")
@patch("os.path.exists", return_value=True)
@patch("os.makedirs")
@patch("shutil.rmtree")
@patch("shutil.move")
@patch.object(EsxImageManager, "_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(EsxImageManager, "_check_image_repair", return_value=True)
@patch("host.hypervisor.esx.image_manager.FileBackedLock")
@raises(DiskAlreadyExistException)
def test_move_image(self, _flock, check_image, _get_ds_type, _mv_dir,
_rmtree, _makedirs, _exists, _delete_file):
# Common case is covered in test_copy_image.
# check that if destination image directory exists we don't call move
# and just bail after removing the tmp dir
_rmtree.reset_mock()
_mv_dir.reset_mock()
expected_tmp_disk_folder = '/vmfs/volumes/ds2/tmp_images/bar'
expected_rm_calls = [call(expected_tmp_disk_folder)]
self.image_manager._move_image("foo", "ds1", expected_tmp_disk_folder)
self.assertEqual(expected_rm_calls, _rmtree.call_args_list)
_makedirs.assert_called_once_with('/vmfs/volumes/ds1/images/fo')
_flock.assert_called_once_with('/vmfs/volumes/ds1/image_foo',
DatastoreType.EXT3, 3)
_delete_file.assert_called_once_with('/vmfs/volumes/ds1/image_foo')
@parameterized.expand([
(True, ),
(False, )
])
@patch("os.path.exists")
@patch.object(EsxImageManager, "_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(EsxImageManager, "_create_image_timestamp_file")
@patch.object(EsxImageManager, "_delete_renamed_image_timestamp_file")
@patch("host.hypervisor.esx.image_manager.FileBackedLock")
def test_validate_existing_image(self,
create,
_flock,
_delete_renamed_timestamp_file,
_create_timestamp_file,
_get_ds_type,
_path_exists):
self._create_image_timestamp_file = create
_path_exists.side_effect = self._local_os_path_exists
_disk_folder = '/vmfs/volumes/ds1/image_foo'
self.image_manager._check_image_repair("foo", "ds1")
if create:
_create_timestamp_file.assert_called_once_with(_disk_folder)
_delete_renamed_timestamp_file.assert_called_once()
else:
assert not _create_timestamp_file.called
assert not _delete_renamed_timestamp_file.called
def _local_os_path_exists(self, pathname):
if not self._create_image_timestamp_file:
return True
if pathname.endswith(EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME):
return False
else:
return True
def test_image_path(self):
image_path = "/vmfs/volumes/ds/image_ttylinux/ttylinux.vmdk"
ds = self.image_manager.get_datastore_id_from_path(image_path)
image = self.image_manager.get_image_id_from_path(image_path)
self.assertEqual(ds, "ds")
self.assertEqual(image, "ttylinux")
@patch.object(EsxImageManager, "_get_datastore_type")
def test_create_image(self, _get_ds_type):
image_id = "image_id"
datastore_id = "ds1"
_get_ds_type.side_effect = (DatastoreType.LOCAL_VMFS, DatastoreType.VSAN)
tmp_image_path = self.image_manager.create_image(image_id, datastore_id)
prefix = "[] /vmfs/volumes/%s/tmp_image_" % datastore_id
self.assertTrue(tmp_image_path.startswith(prefix))
tmp_image_path = self.image_manager.create_image(image_id, datastore_id)
prefix = "[] /vmfs/volumes/%s/image_%s/tmp_image_" % (datastore_id, image_id)
self.assertTrue(tmp_image_path.startswith(prefix))
@patch.object(EsxImageManager, "_move_image")
@patch.object(EsxImageManager, "_create_image_timestamp_file_from_ids")
@patch("os.path.exists")
def test_finalize_image(self, _exists, _create_timestamp, move_image):
# Happy path verify move is called with the right args.
_exists.side_effect = ([True])
self.image_manager.finalize_image("ds1", "[] /vmfs/volumes/ds1/foo", "img_1")
move_image.assert_called_once_with('img_1', 'ds1', '/vmfs/volumes/ds1/foo')
_create_timestamp.assert_called_once_with("ds1", "img_1")
@patch.object(EsxImageManager, "finalize_image")
@patch.object(EsxImageManager, "_manage_disk")
@patch("os.path.exists", return_value=True)
def test_create_image_with_vm_disk(self, _exists, _manage_disk,
_create_image):
vm_disk_path = "/vmfs/volumes/dsname/vms/ab/cd.vmdk"
self.image_manager.create_image_with_vm_disk(
"ds1", "[] /vmfs/volumes/ds1/foo", "img_1", vm_disk_path)
# Verify that we copy the disk correctly
expected_tmp_disk_ds_path = \
"[] /vmfs/volumes/ds1/foo/img_1.vmdk"
_vd_spec = _manage_disk.call_args_list[0][1]['destSpec']
self.assertEqual("thin", _vd_spec.diskType)
self.assertEqual("lsiLogic", _vd_spec.adapterType)
copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName='[] %s' % vm_disk_path,
destName=expected_tmp_disk_ds_path,
destSpec=_vd_spec)
expected_vim_calls = [copy_call]
self.assertEqual(expected_vim_calls, _manage_disk.call_args_list)
_create_image.assert_called_once_with("ds1", "[] /vmfs/volumes/ds1/foo", "img_1")
@patch("shutil.rmtree")
@patch("os.path.exists")
def test_delete_tmp_dir(self, _exists, _rmtree):
self.image_manager.delete_tmp_dir("ds1", "foo")
_exists.assert_called_once("/vmfs/volumes/ds1/foo")
_rmtree.assert_called_once("/vmfs/volumes/ds1/foo")
_exists.reset_mock()
_exists.return_value = False
_rmtree.reset_mock()
self.assertRaises(DirectoryNotFound,
self.image_manager.delete_tmp_dir,
"ds1", "foo")
_exists.assert_called_once("/vmfs/volumes/ds1/foo")
self.assertFalse(_rmtree.called)
def test_image_size(self):
self.ds_manager.image_datastores.return_value = ["ds1", "ds2"]
with patch("host.hypervisor.esx.image_manager.os_vmdk_flat_path"
"") as image_path:
tmpdir = file_util.mkdtemp(delete=True)
image_path.return_value = tmpdir
size = self.image_manager.image_size("image_id")
self.assertTrue(size > 0)
def test_image_size_not_exist(self):
self.ds_manager.image_datastores.return_value = ["ds1", "ds2"]
self.assertRaises(NoSuchResourceException,
self.image_manager.image_size,
"image_id")
class EsxHypervisor(object):
"""Manage ESX Hypervisor."""
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout,
errback=lambda: suicide())
atexit.register(lambda client: client.disconnect(), self.vim_client)
self._uuid = self.vim_client.host_uuid
self.set_memory_overcommit(agent_config.memory_overcommit)
self.datastore_manager = EsxDatastoreManager(
self, agent_config.datastores, agent_config.image_datastores)
# datastore manager needs to update the cache when there is a change.
self.vim_client.add_update_listener(self.datastore_manager)
self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager)
self.disk_manager = EsxDiskManager(self.vim_client,
self.datastore_manager)
self.image_manager = EsxImageManager(self.vim_client,
self.datastore_manager)
self.network_manager = EsxNetworkManager(self.vim_client,
agent_config.networks)
self.system = EsxSystem(self.vim_client)
self.image_manager.monitor_for_cleanup()
self.image_transferer = HttpNfcTransferer(
self.vim_client, self.datastore_manager.image_datastores())
atexit.register(self.image_manager.cleanup)
@property
def uuid(self):
return self._uuid
def check_image(self, image_id, datastore_id):
return self.image_manager.check_image(
image_id, self.datastore_manager.datastore_name(datastore_id))
def acquire_vim_ticket(self):
return self.vim_client.acquire_clone_ticket()
def acquire_cgi_ticket(self, url, op):
return self.vim_client.acquire_cgi_ticket(url, op)
def add_update_listener(self, listener):
self.vim_client.add_update_listener(listener)
def remove_update_listener(self, listener):
self.vim_client.remove_update_listener(listener)
def transfer_image(self, source_image_id, source_datastore,
destination_image_id, destination_datastore, host,
port):
return self.image_transferer.send_image_to_host(
source_image_id, source_datastore, destination_image_id,
destination_datastore, host, port)
def receive_image(self, image_id, datastore, imported_vm_name, metadata,
manifest):
self.image_manager.receive_image(image_id, datastore, imported_vm_name,
metadata, manifest)
def set_memory_overcommit(self, memory_overcommit):
# Enable/Disable large page support. If this host is removed
# from the deployment, large page support will need to be
# explicitly updated by the user.
disable_large_pages = memory_overcommit > 1.0
self.vim_client.set_large_page_support(disable=disable_large_pages)
class ImageSweeperTouchTimestampTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
BASE_TEMP_DIR = "image_sweeper"
IMAGE_TIMESTAMP_FILENAME = EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "image_")
self.dir0 = dir0
# Image dir with correct timestamp file
image_id_1 = str(uuid.uuid4())
dir1 = self.create_dir(image_id_1)
open(os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
# Image dir without the correct timestamp file
image_id_2 = str(uuid.uuid4())
dir2 = self.create_dir(image_id_2)
# Image dir with correct timestamp file
# and with tombstone file
image_id_3 = str(uuid.uuid4())
dir3 = self.create_dir(image_id_3)
open(os.path.join(
dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
self.image_ids = ["", image_id_1, image_id_2, image_id_3]
self.image_dirs = ["", dir1, dir2, dir3]
def tearDown(self):
shutil.rmtree(self.test_dir, True)
@parameterized.expand([
# timestamp_exists
(True, ),
(False, )
])
# The os_vmdk_path method is defined in vm_config.py
# but it is imported in esx/image_manager.py, that is
# the instance we need to patch
@patch("host.hypervisor.esx.image_manager.os_vmdk_path")
def test_touch_timestamp_file(self,
timestamp_exists,
os_vmdk_path):
if not timestamp_exists:
image_index = 2
exception_class = type(OSError())
else:
image_index = 1
exception_class = None
image_id = self.image_ids[image_index]
image_dir = self.image_dirs[image_index]
os_vmdk_path.side_effect = self.patched_os_vmdk_path
timestamp_filename_path = os.path.join(image_dir, self.IMAGE_TIMESTAMP_FILENAME)
pre_mod_time = 0
if timestamp_exists:
# save mod time on the image timestamp file
pre_mod_time = os.path.getmtime(timestamp_filename_path)
try:
time.sleep(1)
self.image_manager.touch_image_timestamp(self.DATASTORE_ID, image_id)
assert_that(exception_class is None)
# check new timestamp
post_mod_time = os.path.getmtime(timestamp_filename_path)
assert_that((post_mod_time > pre_mod_time) is True)
except Exception as ex:
assert_that((type(ex) == exception_class) is True)
def patched_os_vmdk_path(self, datastore, disk_id, folder):
folder = self.dir0
ret = os_vmdk_path(datastore, disk_id, folder)
return ret
def create_dir(self, image_id):
dirname = vm_config.compond_path_join(self.dir0, image_id)
os.makedirs(dirname)
return dirname
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
self.gc_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.deleted = False
self.marker_unlinked = False
# Create various image directories and empty vmdks
image_id_1 = str(uuid.uuid4())
image_id_2 = str(uuid.uuid4())
image_id_3 = str(uuid.uuid4())
image_id_4 = str(uuid.uuid4())
self.image_id_1 = image_id_1
self.image_id_2 = image_id_2
self.image_id_3 = image_id_3
self.image_id_4 = image_id_4
dir1 = os.path.join(self.test_dir, "image_" + image_id_1)
os.makedirs(dir1)
dir2 = os.path.join(self.test_dir, "image_" + image_id_2)
os.makedirs(dir2)
dir3 = os.path.join(self.test_dir, "image_" + image_id_3)
os.makedirs(dir3)
dir4 = os.path.join(self.test_dir, "image_" + image_id_4)
os.makedirs(dir4)
self.marker_file_content_time = 0
self.timestamp_file_mod_time = 0
self.renamed_timestamp_file_mod_time = 0
# Create a good image vmdk under image_id_1,
# also create a valid image marker file
# and a valid timestamp file
vmdk_filename = image_id_1 + ".vmdk"
open(os.path.join(dir1, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
marker_filename = os.path.join(dir1, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_2,
# create timestamp but no image marker file,
vmdk_filename = image_id_2 + ".vmdk"
open(os.path.join(dir2, vmdk_filename), 'w').close()
timestamp_filename = os.path.join(dir2, self.IMAGE_TIMESTAMP_FILENAME)
open(timestamp_filename, 'w').close()
# Create a good image vmdk under image_id_3,
# create image_marker file but no timestamp
# and no renamed timestamp file
vmdk_filename = image_id_3 + ".vmdk"
open(os.path.join(dir3, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir3, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
# Create a good image vmdk under image_id_4,
# create image_marker file, renamed timestamp file
# but no timestamp file
vmdk_filename = image_id_4 + ".vmdk"
open(os.path.join(dir4, vmdk_filename), 'w').close()
marker_filename = os.path.join(dir4, self.IMAGE_MARKER_FILENAME)
open(marker_filename, 'w').close()
timestamp_filename = os.path.join(dir4, self.IMAGE_TIMESTAMP_FILENAME)
renamed_timestamp_filename = timestamp_filename + self.IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX
open(renamed_timestamp_filename, 'w').close()
class TestEsxImageManager(unittest.TestCase):
"""Image Manager tests."""
# We can use even more unit test coverage of the image manager here
@patch.object(VimClient, "acquire_credentials")
@patch.object(VimClient, "update_cache")
@patch("pysdk.connect.Connect")
def setUp(self, connect, update, creds):
creds.return_value = ["username", "password"]
self.vim_client = VimClient(auto_sync=False)
self.ds_manager = MagicMock()
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(self.vim_client, self.ds_manager)
def tearDown(self):
self.vim_client.disconnect(wait=True)
@patch("os.path.isdir", return_value=False)
@patch("os.makedirs", side_effect=OSError)
def test_make_image_dir(self, _makedirs, _isdir):
self.assertRaises(
OSError, self.image_manager._make_image_dir, "ds", "fake_iid")
_isdir.assert_called_once_with("/vmfs/volumes/ds/images/fa/fake_iid")
self.assertEqual(
_makedirs.call_count, EsxImageManager.NUM_MAKEDIRS_ATTEMPTS)
for i in range(0, EsxImageManager.NUM_MAKEDIRS_ATTEMPTS):
self.assertEqual(_makedirs.call_args_list[i][0],
("/vmfs/volumes/ds/images/fa/fake_iid",))
@patch(
"host.hypervisor.esx.image_manager.EsxImageManager.reap_tmp_images")
def test_periodic_reaper(self, mock_reap):
""" Test that the we invoke the image reaper periodically """
image_manager = EsxImageManager(self.vim_client, self.ds_manager)
image_manager.monitor_for_cleanup(reap_interval=0.1)
self.assertFalse(image_manager._image_reaper is None)
retry = 0
while mock_reap.call_count < 2 and retry < 10:
time.sleep(0.1)
retry += 1
image_manager.cleanup()
assert_that(mock_reap.call_count, greater_than(1))
assert_that(retry, is_not(10), "reaper cleanup not called repeatedly")
@parameterized.expand([
(True, ),
(False, )
])
@patch("uuid.uuid4", return_value="fake_id")
@patch("host.hypervisor.esx.vm_config.os_datastore_path")
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_path,
_uuid):
""" Test that stray images are found and deleted by the reaper """
def _fake_ds_folder(datastore, folder):
return "%s__%s" % (datastore, folder)
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_images_folder = _fake_ds_folder(ds.id, TMP_IMAGE_FOLDER_NAME)
tmp_images_dir = os.path.join(tmpdir, tmp_images_folder)
tmp_image_dir = os.path.join(tmp_images_dir, "stray_image")
os.mkdir(tmp_images_dir)
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_path(datastore, folder):
return os.path.join(tmpdir, _fake_ds_folder(datastore, folder))
_os_datastore_path.side_effect = _fake_os_datastore_path
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = EsxImageManager(self.vim_client, ds_manager)
if not _allow_grace_period:
image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0
time.sleep(0.1)
image_manager.reap_tmp_images()
if _allow_grace_period:
# verify stray image is not deleted due to grace period
self.assertTrue(os.path.exists(path))
else:
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
@patch("os.path.isdir")
@patch("os.makedirs")
def test_vmdk_mkdir_eexist(self, _makedirs, _isdir):
eexist = OSError()
eexist.errno = errno.EEXIST
_makedirs.side_effect = eexist
_isdir.side_effect = (False, # dest image dir missing
True) # dest image dir is created
self.image_manager._make_image_dir("ds", "fake_iid")
_isdir.assert_called("/vmfs/volumes/ds/images/fa/fake_iid")
@patch("pysdk.task.WaitForTask")
@patch("uuid.uuid4", return_value="fake_id")
@patch("os.path.exists")
@patch("os.makedirs")
@patch("shutil.copy")
@patch("shutil.rmtree")
@patch("shutil.move")
@patch.object(EsxImageManager, "_manage_disk")
@patch.object(EsxImageManager, "_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(EsxImageManager, "_check_image_repair", return_value=False)
@patch.object(EsxImageManager,
"check_and_validate_image", return_value=False)
@patch.object(EsxImageManager, "_create_image_timestamp_file")
@patch("host.hypervisor.esx.image_manager.FileBackedLock")
def test_copy_image(self, _flock, _create_image_timestamp,
check_image, _check_image_repair,
_get_ds_type, _manage_disk,
_mv_dir, _rmtree, _copy, _makedirs, _exists,
_uuid, _wait_for_task):
_exists.side_effect = (True, # dest image vmdk missing
True, # source meta file present
True) # source manifest file present
self.image_manager.copy_image("ds1", "foo", "ds2", "bar")
os_path_prefix1 = '/vmfs/volumes/ds1/images'
os_path_prefix2 = '/vmfs/volumes/ds2/images'
os_tmp_path_prefix = '/vmfs/volumes/ds2/tmp_images'
assert_that(_copy.call_count, equal_to(2))
_copy.assert_has_calls([
call('%s/fo/foo/foo.%s' % (os_path_prefix1, METADATA_FILE_EXT),
'/vmfs/volumes/ds2/tmp_images/fake_id/bar.%s' %
METADATA_FILE_EXT),
call('%s/fo/foo/foo.%s' % (os_path_prefix1, MANIFEST_FILE_EXT),
'/vmfs/volumes/ds2/tmp_images/fake_id/bar.%s' %
MANIFEST_FILE_EXT),
])
ds_path_prefix1 = '[] ' + os_path_prefix1
ds_tmp_path_prefix = '[] ' + os_tmp_path_prefix
expected_tmp_disk_ds_path = '%s/fake_id/%s.vmdk' % (ds_tmp_path_prefix,
'bar')
_vd_spec = _manage_disk.call_args_list[0][1]['destSpec']
self.assertEqual("thin", _vd_spec.diskType)
self.assertEqual("lsiLogic", _vd_spec.adapterType)
copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName='%s/fo/foo/foo.vmdk' % ds_path_prefix1,
destName=expected_tmp_disk_ds_path,
destSpec=_vd_spec)
expected_vim_calls = [copy_call]
self.assertEqual(expected_vim_calls, _manage_disk.call_args_list)
_mv_dir.assert_called_once_with('/vmfs/volumes/ds2/tmp_images/fake_id',
'%s/ba/bar' % os_path_prefix2)
_create_image_timestamp.assert_called_once_with(
"/vmfs/volumes/ds2/tmp_images/fake_id")
@patch("pysdk.task.WaitForTask")
@patch("uuid.uuid4", return_value="fake_id")
@patch("os.path.exists")
@patch("os.makedirs")
@patch("shutil.copy")
@patch.object(EsxImageManager, "_manage_disk")
@patch.object(EsxImageManager, "_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(EsxImageManager, "check_image", return_value=False)
@patch.object(EsxImageManager, "_create_image_timestamp_file")
@patch("host.hypervisor.esx.image_manager.FileBackedLock")
def test_create_tmp_image(self, _flock, _create_image_timestamp,
check_image, _get_ds_type,
_manage_disk, _copy, _makedirs, _exists,
_uuid, _wait_for_task):
# Common case is the same as the one covered by test_copy_image.
# Check that things work when the src metadata file doesn't exist.
_exists.side_effect = (False, False, True)
ds_path_prefix1 = '[] /vmfs/volumes/ds1/images'
expected_tmp_disk_ds_path = \
"[] /vmfs/volumes/ds2/tmp_images/fake_id/bar.vmdk"
self.image_manager._create_tmp_image("ds1", "foo", "ds2", "bar")
_flock.assert_called_once_with("/vmfs/volumes/ds2/tmp_images/fake_id",
DatastoreType.EXT3)
# Verify that we don't copy the metadata file.
self.assertFalse(_copy.called)
# Verify that we copy the disk correctly
_vd_spec = _manage_disk.call_args_list[0][1]['destSpec']
self.assertEqual("thin", _vd_spec.diskType)
self.assertEqual("lsiLogic", _vd_spec.adapterType)
copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName='%s/fo/foo/foo.vmdk' % ds_path_prefix1,
destName=expected_tmp_disk_ds_path,
destSpec=_vd_spec)
expected_vim_calls = [copy_call]
self.assertEqual(expected_vim_calls, _manage_disk.call_args_list)
# check that we return an IO error if the copy of metadata fails.
_copy.side_effect = IOError
_exists.side_effect = (True, True)
_manage_disk.reset_mock()
_flock.reset_mock()
self.assertRaises(IOError, self.image_manager._create_tmp_image,
"ds1", "foo", "ds2", "bar")
self.assertFalse(_manage_disk.called)
_flock.assert_called_once_with("/vmfs/volumes/ds2/tmp_images/fake_id",
DatastoreType.EXT3)
_create_image_timestamp.assert_called_once_with(
"/vmfs/volumes/ds2/tmp_images/fake_id")
@patch("os.makedirs")
@patch("shutil.rmtree")
@patch("shutil.move")
@patch.object(EsxImageManager, "_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(EsxImageManager, "_check_image_repair", return_value=True)
@patch("host.hypervisor.esx.image_manager.FileBackedLock")
@raises(DiskAlreadyExistException)
def test_move_image(self, _flock, check_image, _get_ds_type, _mv_dir,
_rmtree, _makedirs):
# Common case is covered in test_copy_image.
# check that if destination image directory exists we don't call move
# and just bail after removing the tmp dir
_rmtree.reset_mock()
_mv_dir.reset_mock()
expected_tmp_disk_folder = '/vmfs/volumes/ds2/tmp_images/bar'
expected_rm_calls = [call(expected_tmp_disk_folder)]
self.image_manager._move_image("foo", "ds1", expected_tmp_disk_folder)
self.assertEqual(expected_rm_calls, _rmtree.call_args_list)
_makedirs.assert_called_once_with('/vmfs/volumes/ds1/images/fo')
_flock.assert_called_once_with('/vmfs/volumes/ds1/images/fo/foo',
DatastoreType.EXT3, 3)
@parameterized.expand([
(True, ),
(False, )
])
@patch("os.path.exists")
@patch.object(EsxImageManager, "_get_datastore_type",
return_value=DatastoreType.EXT3)
@patch.object(EsxImageManager, "_create_image_timestamp_file")
@patch.object(EsxImageManager, "_delete_renamed_image_timestamp_file")
@patch("host.hypervisor.esx.image_manager.FileBackedLock")
def test_validate_existing_image(self,
create,
_flock,
_delete_renamed_timestamp_file,
_create_timestamp_file,
_get_ds_type,
_path_exists):
self._create_image_timestamp_file = create
_path_exists.side_effect = self._local_os_path_exists
_disk_folder = '/vmfs/volumes/ds1/images/fo/foo'
self.image_manager._check_image_repair("foo", "ds1")
if create:
_create_timestamp_file.assert_called_once_with(_disk_folder)
_delete_renamed_timestamp_file.assert_called_once()
else:
assert not _create_timestamp_file.called
assert not _delete_renamed_timestamp_file.called
def _local_os_path_exists(self, pathname):
if not self._create_image_timestamp_file:
return True
if pathname.endswith(EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME):
return False
else:
return True
@patch.object(EsxImageManager, "_clean_gc_dir")
@patch.object(EsxImageManager, "_gc_image_dir")
@patch.object(EsxImageManager, "_lock_data_disk")
@patch.object(EsxImageManager, "create_image_tombstone")
@patch.object(EsxImageManager, "check_image_dir")
def test_delete(self, check_image_dir, create_image_tombstone,
lock_data_disk, gc_image_dir, clean_gc_dir):
# Test successful delete
check_image_dir.return_value = True
self.image_manager.delete_image("ds1", "foo", 0, False)
check_image_dir.assert_called_with("foo", "ds1")
create_image_tombstone.assert_called_with("ds1", "foo")
# Test successful delete with force option
self.image_manager.delete_image("ds1", "foo", 0, True)
check_image_dir.assert_called_with("foo", "ds1")
create_image_tombstone.assert_called_with("ds1", "foo")
lock_data_disk.assert_called_with("ds1", "foo")
gc_image_dir.assert_called_with("ds1", "foo")
clean_gc_dir.assert_called()
# Test image not found
check_image_dir.return_value = False
self.assertRaises(ImageNotFoundException,
self.image_manager.delete_image,
"ds1", "foo", 0, False)
@patch("host.hypervisor.esx.image_manager.os_vmdk_path")
@patch("host.hypervisor.esx.image_manager.os_datastore_path")
def test_gc_image_dir(self, dst_path, src_path):
""" Test that we move the directory correctly to the GC location """
src_dir = file_util.mkdtemp(delete=True)
dst_dir = file_util.mkdtemp(delete=True)
src_path.return_value = os.path.join(src_dir, "test.vmdk")
dst_path.return_value = dst_dir
self.image_manager._gc_image_dir("ds1", "foo")
uuid_dir = os.path.join(dst_dir, os.listdir(dst_dir)[0])
# Verify the src directory has been moved into the garbage dir.
self.assertEqual(os.listdir(uuid_dir), [os.path.basename(src_dir)])
src_path.assert_called_once_with("ds1", "foo", IMAGE_FOLDER_NAME)
dst_path.assert_called_once_with("ds1", GC_IMAGE_FOLDER)
def test_image_path(self):
image_path = "/vmfs/volumes/ds/images/tt/ttylinux/ttylinux.vmdk"
ds = self.image_manager.get_datastore_id_from_path(image_path)
image = self.image_manager.get_image_id_from_path(image_path)
self.assertEqual(ds, "ds")
self.assertEqual(image, "ttylinux")
@patch("host.hypervisor.esx.image_manager.os_vmdk_flat_path")
@patch("host.hypervisor.esx.image_manager.os.remove")
def test_lock_data_disk(self, mock_rm, vmdk_flat_path):
""" Test acquisition of the lock on the flat file. """
vmdk_flat_path.return_value = "fake_f_name"
self.assertTrue(self.image_manager._lock_data_disk("ds1", "foo"))
vmdk_flat_path.assert_called_once_with("ds1", "foo")
mock_rm.side_effect = OSError
self.assertFalse(self.image_manager._lock_data_disk("ds1", "foo"))
@parameterized.expand([
("CLOUD", "EAGER", ImageType.CLOUD, ImageReplication.EAGER),
("MANAGEMENT", "EAGER", ImageType.MANAGEMENT, ImageReplication.EAGER),
("CLOUD", "ON_DEMAND", ImageType.CLOUD, ImageReplication.ON_DEMAND),
("MANAGEMENT", "ON_DEMAND", ImageType.MANAGEMENT,
ImageReplication.ON_DEMAND),
])
def test_image_type(self, type, replication, expected_type,
expected_replication):
self.ds_manager.image_datastores.return_value = ["ds1", "ds2"]
with patch("host.hypervisor.esx.image_manager.os_image_manifest_path"
"") as manifest_path:
tmpdir = file_util.mkdtemp(delete=True)
tmpfile = os.path.join(tmpdir, "ds1.manifest")
manifest_path.return_value = tmpfile
with open(tmpfile, 'w+') as f:
f.write('{"imageType":"%s","imageReplication":"%s"}' % (
type, replication))
type, replication = self.image_manager.get_image_manifest(
"image_id")
self.assertEqual(type, expected_type)
self.assertEqual(replication, expected_replication)
def test_image_type_not_exist(self):
self.ds_manager.image_datastores.return_value = ["ds1", "ds2"]
type, replication = self.image_manager.get_image_manifest(
"image_id")
self.assertEqual(type, None)
self.assertEqual(replication, None)
@patch.object(EsxImageManager, "_move_image")
@patch.object(EsxImageManager, "check_image_dir", return_value=False)
@patch.object(EsxImageManager, "_create_image_timestamp_file_from_ids")
@patch("os.path.exists")
def test_create_image(self, _exists, _create_timestamp,
check_image_dir, move_image):
# Happy path verify move is called with the right args.
_exists.side_effect = ([True])
self.image_manager.create_image("ds1", "foo", "img_1")
check_image_dir.assert_called_once_with("img_1", "ds1")
move_image.assert_called_once_with('img_1', 'ds1',
'/vmfs/volumes/ds1/foo')
_create_timestamp.assert_called_once_with("ds1", "img_1")
# Verify error if tmp image doesn't exist
_exists.side_effect = ([False])
move_image.reset_mock()
self.assertRaises(ImageNotFoundException,
self.image_manager.create_image,
"ds1", "foo", "img_1")
self.assertFalse(move_image.called)
# Verify error if destination image already exists.
_exists.side_effect = ([True])
move_image.reset_mock()
check_image_dir.return_value = True
self.assertRaises(DiskAlreadyExistException,
self.image_manager.create_image,
"ds1", "foo", "img_1")
self.assertFalse(move_image.called)
@patch.object(EsxImageManager, "create_image")
@patch.object(EsxImageManager, "_manage_disk")
@patch("os.path.exists", return_value=True)
def test_create_image_with_vm_disk(self, _exists, _manage_disk,
_create_image):
vm_disk_path = "/vmfs/volumes/dsname/vms/ab/cd.vmdk"
self.image_manager.create_image_with_vm_disk(
"ds1", "foo", "img_1", vm_disk_path)
# Verify that we copy the disk correctly
expected_tmp_disk_ds_path = \
"[] /vmfs/volumes/ds1/foo/img_1.vmdk"
_vd_spec = _manage_disk.call_args_list[0][1]['destSpec']
self.assertEqual("thin", _vd_spec.diskType)
self.assertEqual("lsiLogic", _vd_spec.adapterType)
copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName='[] %s' % vm_disk_path,
destName=expected_tmp_disk_ds_path,
destSpec=_vd_spec)
expected_vim_calls = [copy_call]
self.assertEqual(expected_vim_calls, _manage_disk.call_args_list)
_create_image.assert_called_once_with("ds1", "foo", "img_1")
@patch("shutil.rmtree")
@patch("os.path.exists")
def test_delete_tmp_dir(self, _exists, _rmtree):
self.image_manager.delete_tmp_dir("ds1", "foo")
_exists.assert_called_once("/vmfs/volumes/ds1/foo")
_rmtree.assert_called_once("/vmfs/volumes/ds1/foo")
_exists.reset_mock()
_exists.return_value = False
_rmtree.reset_mock()
self.assertRaises(DirectoryNotFound,
self.image_manager.delete_tmp_dir,
"ds1", "foo")
_exists.assert_called_once("/vmfs/volumes/ds1/foo")
self.assertFalse(_rmtree.called)
def test_image_size(self):
self.ds_manager.image_datastores.return_value = ["ds1", "ds2"]
with patch("host.hypervisor.esx.image_manager.os_vmdk_flat_path"
"") as image_path:
tmpdir = file_util.mkdtemp(delete=True)
image_path.return_value = tmpdir
size = self.image_manager.image_size("image_id")
self.assertTrue(size > 0)
def test_image_size_not_exist(self):
self.ds_manager.image_datastores.return_value = ["ds1", "ds2"]
self.assertRaises(NoSuchResourceException,
self.image_manager.image_size,
"image_id")
class ImageSweeperTouchTimestampTestCase(unittest.TestCase):
DATASTORE_ID = "DS01"
BASE_TEMP_DIR = "image_sweeper"
IMAGE_TIMESTAMP_FILENAME = EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME
IMAGE_TOMBSTONE_FILENAME = EsxImageManager.IMAGE_TOMBSTONE_FILE_NAME
def setUp(self):
self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR)
services.register(ServiceName.AGENT_CONFIG, MagicMock())
self.image_manager = EsxImageManager(MagicMock(), MagicMock())
self.vm_manager = MagicMock()
self.image_sweeper = DatastoreImageSweeper(self.image_manager,
self.DATASTORE_ID)
self.image_sweeper._task_runner = MagicMock()
self.image_sweeper._task_runner.is_stopped.return_value = False
self.delete_count = 0
# Create various image directories and empty vmdks
dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "images/im")
self.dir0 = dir0
# Image dir with correct timestamp file
image_id_1 = str(uuid.uuid4())
dir1 = self.create_dir(image_id_1)
open(os.path.join(
dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
# Image dir without the correct timestamp file
image_id_2 = str(uuid.uuid4())
dir2 = self.create_dir(image_id_2)
# Image dir with correct timestamp file
# and with tombstone file
image_id_3 = str(uuid.uuid4())
dir3 = self.create_dir(image_id_3)
open(os.path.join(
dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close()
open(os.path.join(
dir3, self.IMAGE_TOMBSTONE_FILENAME), 'w').close()
self.image_ids = ["", image_id_1, image_id_2, image_id_3]
self.image_dirs = ["", dir1, dir2, dir3]
def tearDown(self):
shutil.rmtree(self.test_dir, True)
@parameterized.expand([
# timestamp_exists, tombstone_exists
(True, False),
(True, True),
(False, False)
])
# The os_vmdk_path method is defined in vm_config.py
# but it is imported in esx/image_manager.py, that is
# the instance we need to patch
@patch("host.hypervisor.esx.image_manager.os_vmdk_path")
def test_touch_timestamp_file(self,
timestamp_exists,
tombstone_exists,
os_vmdk_path):
if tombstone_exists:
image_index = 3
exception_class = type(InvalidImageState())
elif not timestamp_exists:
image_index = 2
exception_class = type(OSError())
else:
image_index = 1
exception_class = None
image_id = self.image_ids[image_index]
image_dir = self.image_dirs[image_index]
os_vmdk_path.side_effect = self.patched_os_vmdk_path
timestamp_filename_path = \
os.path.join(image_dir, self.IMAGE_TIMESTAMP_FILENAME)
pre_mod_time = 0
if timestamp_exists:
# save mod time on the image timestamp file
pre_mod_time = os.path.getmtime(timestamp_filename_path)
try:
time.sleep(1)
self.image_manager.\
touch_image_timestamp(self.DATASTORE_ID,
image_id)
assert_that(exception_class is None)
# check new timestamp
post_mod_time = os.path.getmtime(timestamp_filename_path)
assert_that((post_mod_time > pre_mod_time) is True)
except Exception as ex:
assert_that((type(ex) == exception_class) is True)
@patch("host.hypervisor.esx.image_manager.os_vmdk_path")
def test_create_tombstone_file(self,
os_vmdk_path):
image_index = 1
image_id = self.image_ids[image_index]
image_dir = self.image_dirs[image_index]
os_vmdk_path.side_effect = self.patched_os_vmdk_path
tombstone_filename_path = \
os.path.join(image_dir, self.IMAGE_TOMBSTONE_FILENAME)
self.image_manager.create_image_tombstone(self.DATASTORE_ID,
image_id)
# check tombstone exists
exists = os.path.exists(tombstone_filename_path)
assert_that(exists is True)
def patched_os_vmdk_path(self, datastore, disk_id, folder):
folder = self.dir0
ret = os_vmdk_path(datastore, disk_id, folder)
return ret
def create_dir(self, image_id):
dirname = os.path.join(self.dir0,
_disk_path(image_id))
os.makedirs(dirname)
return dirname
class EsxHypervisor(object):
"""Manage ESX Hypervisor."""
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout,
errback=lambda: suicide())
atexit.register(lambda client: client.disconnect(), self.vim_client)
self._uuid = self.vim_client.host_uuid
self.datastore_manager = EsxDatastoreManager(
self, agent_config.datastores, agent_config.image_datastores)
# datastore manager needs to update the cache when there is a change.
self.vim_client.add_update_listener(self.datastore_manager)
self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager)
self.disk_manager = EsxDiskManager(self.vim_client,
self.datastore_manager)
self.image_manager = EsxImageManager(self.vim_client,
self.datastore_manager)
self.network_manager = EsxNetworkManager(self.vim_client,
agent_config.networks)
self.system = EsxSystem(self.vim_client)
self.image_manager.monitor_for_cleanup()
self.image_transferer = HttpNfcTransferer(
self.vim_client,
self.datastore_manager.image_datastores())
atexit.register(self.image_manager.cleanup)
@property
def uuid(self):
return self._uuid
def check_image(self, image_id, datastore_id):
return self.image_manager.check_image(
image_id, self.datastore_manager.datastore_name(datastore_id)
)
def acquire_vim_ticket(self):
return self.vim_client.acquire_clone_ticket()
def add_update_listener(self, listener):
self.vim_client.add_update_listener(listener)
def remove_update_listener(self, listener):
self.vim_client.remove_update_listener(listener)
def transfer_image(self, source_image_id, source_datastore,
destination_image_id, destination_datastore,
host, port):
return self.image_transferer.send_image_to_host(
source_image_id, source_datastore,
destination_image_id, destination_datastore, host, port)
def prepare_receive_image(self, image_id, datastore):
return self.image_manager.prepare_receive_image(image_id, datastore)
def receive_image(self, image_id, datastore, imported_vm_name, metadata):
self.image_manager.receive_image(image_id, datastore, imported_vm_name, metadata)
def set_memory_overcommit(self, memory_overcommit):
# Enable/Disable large page support. If this host is removed
# from the deployment, large page support will need to be
# explicitly updated by the user.
disable_large_pages = memory_overcommit > 1.0
self.vim_client.set_large_page_support(disable=disable_large_pages)
class EsxHypervisor(object):
"""Manage ESX Hypervisor."""
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
errback = lambda: suicide()
self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout,
errback=errback)
atexit.register(lambda client: client.disconnect(), self.vim_client)
self._uuid = self.vim_client.host_uuid
# Enable/Disable large page support. If this host is removed
# from the deployment, large page support will need to be
# explicitly updated by the user.
disable_large_pages = agent_config.memory_overcommit > 1.0
self.vim_client.set_large_page_support(disable=disable_large_pages)
image_datastores = [ds["name"] for ds in agent_config.image_datastores]
self.datastore_manager = EsxDatastoreManager(self,
agent_config.datastores,
image_datastores)
# datastore manager needs to update the cache when there is a change.
self.vim_client.add_update_listener(self.datastore_manager)
self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager)
self.disk_manager = EsxDiskManager(self.vim_client,
self.datastore_manager)
self.image_manager = EsxImageManager(self.vim_client,
self.datastore_manager)
self.network_manager = EsxNetworkManager(self.vim_client,
agent_config.networks)
self.system = EsxSystem(self.vim_client)
self.image_manager.monitor_for_cleanup()
self.image_transferer = HttpNfcTransferer(self.vim_client,
image_datastores)
atexit.register(self.image_manager.cleanup)
@property
def uuid(self):
return self._uuid
@property
def config(self):
config = gen.hypervisor.esx.ttypes.EsxConfig()
return TSerialization.serialize(config)
def normalized_load(self):
""" Return the maximum of the normalized memory/cpu loads"""
memory = self.system.memory_info()
memory_load = memory.used * 100 / memory.total
# get average cpu load percentage in past 20 seconds
# since hostd takes a sample in every 20 seconds
# we use the min 20secs here to get the latest
# CPU active average over 1 minute
host_stats = copy.copy(self.vim_client.get_perf_manager_stats(20))
cpu_load = host_stats['rescpu.actav1'] / 100
return max(memory_load, cpu_load)
def check_image(self, image_id, datastore_id):
return self.image_manager.check_image(
image_id, self.datastore_manager.datastore_name(datastore_id))
def acquire_vim_ticket(self):
return self.vim_client.acquire_clone_ticket()
def acquire_cgi_ticket(self, url, op):
return self.vim_client.acquire_cgi_ticket(url, op)
def add_update_listener(self, listener):
self.vim_client.add_update_listener(listener)
def remove_update_listener(self, listener):
self.vim_client.remove_update_listener(listener)
def transfer_image(self, source_image_id, source_datastore,
destination_image_id, destination_datastore, host,
port):
return self.image_transferer.send_image_to_host(
source_image_id, source_datastore, destination_image_id,
destination_datastore, host, port)
def receive_image(self, image_id, datastore, imported_vm_name):
self.image_manager.receive_image(image_id, datastore, imported_vm_name)
class EsxHypervisor(object):
"""Manage ESX Hypervisor."""
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
errback = lambda: suicide()
self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout,
errback=errback)
atexit.register(lambda client: client.disconnect(), self.vim_client)
self._uuid = self.vim_client.host_uuid
self.set_memory_overcommit(agent_config.memory_overcommit)
image_datastores = [ds["name"] for ds in agent_config.image_datastores]
self.datastore_manager = EsxDatastoreManager(
self, agent_config.datastores, agent_config.image_datastores)
# datastore manager needs to update the cache when there is a change.
self.vim_client.add_update_listener(self.datastore_manager)
self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager)
self.disk_manager = EsxDiskManager(self.vim_client,
self.datastore_manager)
self.image_manager = EsxImageManager(self.vim_client,
self.datastore_manager)
self.network_manager = EsxNetworkManager(self.vim_client,
agent_config.networks)
self.system = EsxSystem(self.vim_client)
self.image_manager.monitor_for_cleanup()
self.image_transferer = HttpNfcTransferer(self.vim_client,
image_datastores)
atexit.register(self.image_manager.cleanup)
@property
def uuid(self):
return self._uuid
@property
def config(self):
config = gen.hypervisor.esx.ttypes.EsxConfig()
return TSerialization.serialize(config)
def normalized_load(self):
""" Return the maximum of the normalized memory/cpu loads"""
memory = self.system.memory_info()
memory_load = memory.used * 100 / memory.total
# get average cpu load percentage in past 20 seconds
# since hostd takes a sample in every 20 seconds
# we use the min 20secs here to get the latest
# CPU active average over 1 minute
host_stats = copy.copy(self.vim_client.get_perf_manager_stats(20))
cpu_load = host_stats['rescpu.actav1'] / 100
return max(memory_load, cpu_load)
def check_image(self, image_id, datastore_id):
return self.image_manager.check_image(
image_id, self.datastore_manager.datastore_name(datastore_id)
)
def acquire_vim_ticket(self):
return self.vim_client.acquire_clone_ticket()
def acquire_cgi_ticket(self, url, op):
return self.vim_client.acquire_cgi_ticket(url, op)
def add_update_listener(self, listener):
self.vim_client.add_update_listener(listener)
def remove_update_listener(self, listener):
self.vim_client.remove_update_listener(listener)
def transfer_image(self, source_image_id, source_datastore,
destination_image_id, destination_datastore,
host, port):
return self.image_transferer.send_image_to_host(
source_image_id, source_datastore,
destination_image_id, destination_datastore, host, port)
def receive_image(self, image_id, datastore, imported_vm_name):
self.image_manager.receive_image(
image_id, datastore, imported_vm_name)
def set_memory_overcommit(self, memory_overcommit):
# Enable/Disable large page support. If this host is removed
# from the deployment, large page support will need to be
# explicitly updated by the user.
disable_large_pages = memory_overcommit > 1.0
self.vim_client.set_large_page_support(disable=disable_large_pages)
