def setUp(self): self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.vm_manager = MagicMock() self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID) self.image_sweeper._task_runner = MagicMock() self.image_sweeper._task_runner.is_stopped.return_value = False self.delete_count = 0 # Create various image directories and empty vmdks dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "image_") self.dir0 = dir0 # Image dir with correct timestamp file image_id_1 = str(uuid.uuid4()) dir1 = self.create_dir(image_id_1) open(os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close() # Image dir without the correct timestamp file image_id_2 = str(uuid.uuid4()) dir2 = self.create_dir(image_id_2) # Image dir with correct timestamp file # and with tombstone file image_id_3 = str(uuid.uuid4()) dir3 = self.create_dir(image_id_3) open(os.path.join( dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close() self.image_ids = ["", image_id_1, image_id_2, image_id_3] self.image_dirs = ["", dir1, dir2, dir3]
def __init__(self, agent_config): self.logger = logging.getLogger(__name__) # If VimClient's housekeeping thread failed to update its own cache, # call errback to commit suicide. Watchdog will bring up the agent # again. self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout, errback=lambda: suicide()) atexit.register(lambda client: client.disconnect(), self.vim_client) self._uuid = self.vim_client.host_uuid self.set_memory_overcommit(agent_config.memory_overcommit) self.datastore_manager = EsxDatastoreManager( self, agent_config.datastores, agent_config.image_datastores) # datastore manager needs to update the cache when there is a change. self.vim_client.add_update_listener(self.datastore_manager) self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager) self.disk_manager = EsxDiskManager(self.vim_client, self.datastore_manager) self.image_manager = EsxImageManager(self.vim_client, self.datastore_manager) self.network_manager = EsxNetworkManager(self.vim_client, agent_config.networks) self.system = EsxSystem(self.vim_client) self.image_manager.monitor_for_cleanup() self.image_transferer = HttpNfcTransferer( self.vim_client, self.datastore_manager.image_datastores()) atexit.register(self.image_manager.cleanup)
def test_reap_tmp_images(self, _os_datastore_path, _uuid): """ Test that stray images are found and deleted by the reaper """ def _fake_ds_folder(datastore, folder): return "%s__%s" % (datastore, folder) ds = MagicMock() ds.id = "dsid" ds.type = DatastoreType.EXT3 # In a random transient directory, set up a directory to act as the # tmp images folder and to contain a stray image folder with a file. tmpdir = file_util.mkdtemp(delete=True) tmp_images_folder = _fake_ds_folder(ds.id, TMP_IMAGE_FOLDER_NAME) tmp_images_dir = os.path.join(tmpdir, tmp_images_folder) tmp_image_dir = os.path.join(tmp_images_dir, "stray_image") os.mkdir(tmp_images_dir) os.mkdir(tmp_image_dir) (fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir) self.assertTrue(os.path.exists(path)) def _fake_os_datastore_path(datastore, folder): return os.path.join(tmpdir, _fake_ds_folder(datastore, folder)) _os_datastore_path.side_effect = _fake_os_datastore_path ds_manager = MagicMock() ds_manager.get_datastores.return_value = [ds] image_manager = EsxImageManager(self.vim_client, ds_manager) image_manager.reap_tmp_images() # verify stray image is deleted self.assertFalse(os.path.exists(path))
def test_periodic_reaper(self, mock_reap): """ Test that the we invoke the image reaper periodically """ image_manager = EsxImageManager(self.vim_client, self.ds_manager) image_manager.monitor_for_cleanup(reap_interval=0.1) self.assertFalse(image_manager._image_reaper is None) retry = 0 while mock_reap.call_count < 2 and retry < 10: time.sleep(0.1) retry += 1 image_manager.cleanup() assert_that(mock_reap.call_count, greater_than(1)) assert_that(retry, is_not(10), "reaper cleanup not called repeatedly")
def __init__(self, agent_config): self.logger = logging.getLogger(__name__) # If VimClient's housekeeping thread failed to update its own cache, # call errback to commit suicide. Watchdog will bring up the agent # again. errback = lambda: suicide() self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout, errback=errback) atexit.register(lambda client: client.disconnect(), self.vim_client) self._uuid = self.vim_client.host_uuid # Enable/Disable large page support. If this host is removed # from the deployment, large page support will need to be # explicitly updated by the user. disable_large_pages = agent_config.memory_overcommit > 1.0 self.vim_client.set_large_page_support(disable=disable_large_pages) image_datastores = [ds["name"] for ds in agent_config.image_datastores] self.datastore_manager = EsxDatastoreManager( self, agent_config.datastores, image_datastores) # datastore manager needs to update the cache when there is a change. self.vim_client.add_update_listener(self.datastore_manager) self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager) self.disk_manager = EsxDiskManager(self.vim_client, self.datastore_manager) self.image_manager = EsxImageManager(self.vim_client, self.datastore_manager) self.network_manager = EsxNetworkManager(self.vim_client, agent_config.networks) self.system = EsxSystem(self.vim_client) self.image_manager.monitor_for_cleanup() atexit.register(self.image_manager.cleanup)
def __init__(self, agent_config): self.logger = logging.getLogger(__name__) # If VimClient's housekeeping thread failed to update its own cache, # call errback to commit suicide. Watchdog will bring up the agent # again. errback = lambda: suicide() self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout, errback=errback) atexit.register(lambda client: client.disconnect(), self.vim_client) self._uuid = self.vim_client.host_uuid self.set_memory_overcommit(agent_config.memory_overcommit) image_datastores = [ds["name"] for ds in agent_config.image_datastores] self.datastore_manager = EsxDatastoreManager( self, agent_config.datastores, agent_config.image_datastores) # datastore manager needs to update the cache when there is a change. self.vim_client.add_update_listener(self.datastore_manager) self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager) self.disk_manager = EsxDiskManager(self.vim_client, self.datastore_manager) self.image_manager = EsxImageManager(self.vim_client, self.datastore_manager) self.network_manager = EsxNetworkManager(self.vim_client, agent_config.networks) self.system = EsxSystem(self.vim_client) self.image_manager.monitor_for_cleanup() self.image_transferer = HttpNfcTransferer(self.vim_client, image_datastores) atexit.register(self.image_manager.cleanup)
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_path, _uuid): """ Test that stray images are found and deleted by the reaper """ def _fake_ds_folder(datastore, folder): return "%s/%s" % (datastore, folder) ds = MagicMock() ds.id = "dsid" ds.type = DatastoreType.EXT3 # In a random transient directory, set up a directory to act as the # tmp images folder and to contain a stray image folder with a file. tmpdir = file_util.mkdtemp(delete=True) tmp_ds_dir = os.path.join(tmpdir, ds.id) os.mkdir(tmp_ds_dir) tmp_image_dir = os.path.join( tmp_ds_dir, compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image")) os.mkdir(tmp_image_dir) (fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir) self.assertTrue(os.path.exists(path)) def _fake_os_datastore_path(datastore, folder): return os.path.join(tmpdir, _fake_ds_folder(datastore, folder)) _os_datastore_path.side_effect = _fake_os_datastore_path ds_manager = MagicMock() ds_manager.get_datastores.return_value = [ds] image_manager = EsxImageManager(self.vim_client, ds_manager) if not _allow_grace_period: image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0 time.sleep(0.1) image_manager.reap_tmp_images() if _allow_grace_period: # verify stray image is not deleted due to grace period self.assertTrue(os.path.exists(path)) else: # verify stray image is deleted self.assertFalse(os.path.exists(path))
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_path, _uuid): """ Test that stray images are found and deleted by the reaper """ def _fake_ds_folder(datastore, folder): return "%s/%s" % (datastore, folder) ds = MagicMock() ds.id = "dsid" ds.type = DatastoreType.EXT3 # In a random transient directory, set up a directory to act as the # tmp images folder and to contain a stray image folder with a file. tmpdir = file_util.mkdtemp(delete=True) tmp_ds_dir = os.path.join(tmpdir, ds.id) os.mkdir(tmp_ds_dir) tmp_image_dir = os.path.join(tmp_ds_dir, compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image")) os.mkdir(tmp_image_dir) (fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir) self.assertTrue(os.path.exists(path)) def _fake_os_datastore_path(datastore, folder): return os.path.join(tmpdir, _fake_ds_folder(datastore, folder)) _os_datastore_path.side_effect = _fake_os_datastore_path ds_manager = MagicMock() ds_manager.get_datastores.return_value = [ds] image_manager = EsxImageManager(self.vim_client, ds_manager) if not _allow_grace_period: image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0 time.sleep(0.1) image_manager.reap_tmp_images() if _allow_grace_period: # verify stray image is not deleted due to grace period self.assertTrue(os.path.exists(path)) else: # verify stray image is deleted self.assertFalse(os.path.exists(path))
def setUp(self): self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.vm_manager = MagicMock() self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID) self.image_sweeper._task_runner = MagicMock() self.image_sweeper._task_runner.is_stopped.return_value = False self.delete_count = 0 # Create various image directories and empty vmdks dir0 = os.path.join(self.test_dir, "images/im") image_id_1 = str(uuid.uuid4()) image_id_2 = str(uuid.uuid4()) image_id_3 = str(uuid.uuid4()) image_id_4 = "invalid_image_id" self.image_ids = ["", image_id_1, image_id_2, image_id_3, image_id_4] dir1 = os.path.join(dir0, image_id_1) os.makedirs(dir1) dir2 = os.path.join(dir0, image_id_2) os.makedirs(dir2) dir3 = os.path.join(dir0, image_id_3) os.makedirs(dir3) dir4 = os.path.join(dir0, image_id_4) os.makedirs(dir4) # Create a vmdk under "im", since the image_id is # not a valid uuid it should be skipped open(os.path.join(dir0, "im.vmdk"), 'w').close() # Create a good image vmdk under image_id_1 but # no image marker file, this should not be deleted vmdk_filename = image_id_1 + ".vmdk" open(os.path.join(dir1, vmdk_filename), 'w').close() # Create a good image vmdk under image_id_2, also create # an unused image marker file, image_id_2 should be # deleted vmdk_filename = image_id_2 + ".vmdk" open(os.path.join(dir2, vmdk_filename), 'w').close() open(os.path.join(dir2, self.IMAGE_MARKER_FILENAME), 'w').close() # Create a marker file under dir3 but no # vmdk file. It should be deleted as well open(os.path.join(dir3, self.IMAGE_MARKER_FILENAME), 'w').close() # Create a vmdk under an invalid image directory, # also create a marker file. Since the image_id # is not valid it should not be deleted vmdk_filename = image_id_4 + ".vmdk" open(os.path.join(dir4, vmdk_filename), 'w').close() open(os.path.join(dir4, self.IMAGE_MARKER_FILENAME), 'w').close()
def setUp(self): self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.vm_manager = MagicMock() self.image_scanner = DatastoreImageScanner(self.image_manager, self.vm_manager, self.DATASTORE_ID) self.image_scanner._task_runner = MagicMock() self.image_scanner._task_runner.is_stopped.return_value = False self.write_count = 0 # Create various image directories and empty vmdks dir0 = os.path.join(self.test_dir, "images/im") image_id_1 = str(uuid.uuid4()) image_id_2 = str(uuid.uuid4()) image_id_3 = str(uuid.uuid4()) image_id_4 = "invalid_image_id" self.image_ids = ["", image_id_1, image_id_2, image_id_3, image_id_4] dir1 = os.path.join(dir0, image_id_1) os.makedirs(dir1) dir2 = os.path.join(dir0, image_id_2) os.makedirs(dir2) dir3 = os.path.join(dir0, image_id_3) os.makedirs(dir3) dir4 = os.path.join(dir0, image_id_4) os.makedirs(dir4) # Create a vmdk under "im", since the image_id is # not a valid uuid it should be skipped open(os.path.join(dir0, "im.vmdk"), 'w').close() # Create a good image vmdk under image_id_1, the name # of the vmdk matches the directory that contains it # so this is a valid image to remove vmdk_filename = image_id_1 + ".vmdk" open(os.path.join(dir1, vmdk_filename), 'w').close() # Create a good image vmdk under image_id_2, also create # an unused image marker file, image_id_2 should also be # included in the list of images to remove vmdk_filename = image_id_2 + ".vmdk" open(os.path.join(dir2, vmdk_filename), 'w').close() open(os.path.join(dir2, self.image_manager.IMAGE_MARKER_FILE_NAME), 'w').close() # Don't create anything under directory dir3 # it should still mark the image as deletable # Create a vmdk under an invalid image directory, since # the image id is not valid it should not mark it # for deletion vmdk_filename = image_id_4 + ".vmdk" open(os.path.join(dir4, vmdk_filename), 'w').close()
def __init__(self, agent_config): self.logger = logging.getLogger(__name__) # If VimClient's housekeeping thread failed to update its own cache, # call errback to commit suicide. Watchdog will bring up the agent # again. errback = lambda: suicide() self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout, errback=errback) atexit.register(lambda client: client.disconnect(), self.vim_client) self._uuid = self.vim_client.host_uuid # Enable/Disable large page support. If this host is removed # from the deployment, large page support will need to be # explicitly updated by the user. disable_large_pages = agent_config.memory_overcommit > 1.0 self.vim_client.set_large_page_support(disable=disable_large_pages) image_datastores = [ds["name"] for ds in agent_config.image_datastores] self.datastore_manager = EsxDatastoreManager(self, agent_config.datastores, image_datastores) # datastore manager needs to update the cache when there is a change. self.vim_client.add_update_listener(self.datastore_manager) self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager) self.disk_manager = EsxDiskManager(self.vim_client, self.datastore_manager) self.image_manager = EsxImageManager(self.vim_client, self.datastore_manager) self.network_manager = EsxNetworkManager(self.vim_client, agent_config.networks) self.system = EsxSystem(self.vim_client) self.image_manager.monitor_for_cleanup() self.image_transferer = HttpNfcTransferer(self.vim_client, image_datastores) atexit.register(self.image_manager.cleanup)
def setUp(self, connect, update, creds): # Create VM manager creds.return_value = ["username", "password"] self.vim_client = VimClient(auto_sync=False) self.vim_client.wait_for_task = MagicMock() self.patcher = patch("host.hypervisor.esx.vm_config.GetEnv") self.patcher.start() self.vm_manager = EsxVmManager(self.vim_client, MagicMock()) services.register(ServiceName.AGENT_CONFIG, MagicMock()) # Set up test files self.base_dir = os.path.dirname(__file__) self.test_dir = os.path.join(self.base_dir, "../../test_files") self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.image_scanner = DatastoreImageScanner(self.image_manager, self.vm_manager, self.DATASTORE_ID) self.write_count = 0
def setUp(self): self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.vm_manager = MagicMock() self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID) self.image_sweeper._task_runner = MagicMock() self.image_sweeper._task_runner.is_stopped.return_value = False self.delete_count = 0 # Create various image directories and empty vmdks dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "images/im") self.dir0 = dir0 # Image dir with correct timestamp file image_id_1 = str(uuid.uuid4()) dir1 = self.create_dir(image_id_1) open(os.path.join( dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close() # Image dir without the correct timestamp file image_id_2 = str(uuid.uuid4()) dir2 = self.create_dir(image_id_2) # Image dir with correct timestamp file # and with tombstone file image_id_3 = str(uuid.uuid4()) dir3 = self.create_dir(image_id_3) open(os.path.join( dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close() open(os.path.join( dir3, self.IMAGE_TOMBSTONE_FILENAME), 'w').close() self.image_ids = ["", image_id_1, image_id_2, image_id_3] self.image_dirs = ["", dir1, dir2, dir3]
class ImageSweeperDeleteSingleImageTestCase(unittest.TestCase): DATASTORE_ID = "DS01" BASE_TEMP_DIR = "delete_single_image" IMAGE_MARKER_FILENAME = EsxImageManager.IMAGE_MARKER_FILE_NAME IMAGE_TIMESTAMP_FILENAME = EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX = EsxImageManager.IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX def setUp(self): self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) self.gc_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.vm_manager = MagicMock() self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID) self.deleted = False self.marker_unlinked = False # Create various image directories and empty vmdks image_id_1 = str(uuid.uuid4()) image_id_2 = str(uuid.uuid4()) image_id_3 = str(uuid.uuid4()) image_id_4 = str(uuid.uuid4()) self.image_id_1 = image_id_1 self.image_id_2 = image_id_2 self.image_id_3 = image_id_3 self.image_id_4 = image_id_4 dir1 = os.path.join(self.test_dir, "image_" + image_id_1) os.makedirs(dir1) dir2 = os.path.join(self.test_dir, "image_" + image_id_2) os.makedirs(dir2) dir3 = os.path.join(self.test_dir, "image_" + image_id_3) os.makedirs(dir3) dir4 = os.path.join(self.test_dir, "image_" + image_id_4) os.makedirs(dir4) self.marker_file_content_time = 0 self.timestamp_file_mod_time = 0 self.renamed_timestamp_file_mod_time = 0 # Create a good image vmdk under image_id_1, # also create a valid image marker file # and a valid timestamp file vmdk_filename = image_id_1 + ".vmdk" open(os.path.join(dir1, vmdk_filename), 'w').close() timestamp_filename = os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME) open(timestamp_filename, 'w').close() marker_filename = os.path.join(dir1, self.IMAGE_MARKER_FILENAME) open(marker_filename, 'w').close() # Create a good image vmdk under image_id_2, # create timestamp but no image marker file, vmdk_filename = image_id_2 + ".vmdk" open(os.path.join(dir2, vmdk_filename), 'w').close() timestamp_filename = os.path.join(dir2, self.IMAGE_TIMESTAMP_FILENAME) open(timestamp_filename, 'w').close() # Create a good image vmdk under image_id_3, # create image_marker file but no timestamp # and no renamed timestamp file vmdk_filename = image_id_3 + ".vmdk" open(os.path.join(dir3, vmdk_filename), 'w').close() marker_filename = os.path.join(dir3, self.IMAGE_MARKER_FILENAME) open(marker_filename, 'w').close() # Create a good image vmdk under image_id_4, # create image_marker file, renamed timestamp file # but no timestamp file vmdk_filename = image_id_4 + ".vmdk" open(os.path.join(dir4, vmdk_filename), 'w').close() marker_filename = os.path.join(dir4, self.IMAGE_MARKER_FILENAME) open(marker_filename, 'w').close() timestamp_filename = os.path.join(dir4, self.IMAGE_TIMESTAMP_FILENAME) renamed_timestamp_filename = timestamp_filename + self.IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX open(renamed_timestamp_filename, 'w').close() def tearDown(self): shutil.rmtree(self.test_dir, True) shutil.rmtree(self.gc_dir, True) # The following test plays with the content of # marker file (a timestamp) and the mod time of # the image timestamp file before and after rename. # It should delete the image only if # marker - grace period > timestamp AND # timestamp == timestamp after rename # All the other cases should point to the fact # that the image has been used after the image # scan started. To avoid problems due to # non synchronized clocks on different hosts # a grace period of 10 minutes is applied on # the timestamp from the marker file @parameterized.expand([ # marker time, mod time, mod time after rename, deleted (1061, 1000, 1000, True), (1060, 1000, 1000, False), (1000, 1000, 1000, False), (1000, 1001, 1001, False), (2000, 1000, 2010, False) ]) @patch("host.hypervisor.esx.image_manager.EsxImageManager._read_marker_file") @patch("host.hypervisor.esx.image_manager.EsxImageManager._get_datastore_type") @patch("host.hypervisor.esx.image_manager.EsxImageManager._get_mod_time") @patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rename") @patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_unlink") @patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rm_rf") def test_delete_single_image( self, marker_file_content_time, timestamp_file_mod_time, renamed_timestamp_file_mod_time, deleted, rm_rf, unlink, rename, get_mod_time, get_datastore_type, read_marker_file): self.marker_file_content_time = marker_file_content_time self.timestamp_file_mod_time = timestamp_file_mod_time self.renamed_timestamp_file_mod_time = renamed_timestamp_file_mod_time marker_unlinked = not deleted read_marker_file.side_effect = self.patched_read_marker_file get_datastore_type.side_effect = self.patched_get_datastore_type get_mod_time.side_effect = self.patched_get_mod_time rename.side_effect = self.patched_rename unlink.side_effect = self.patched_unlink rm_rf.side_effect = self.patched_rm_rf good_dir = os.path.join(self.test_dir, "image_" + self.image_id_1) ret = self.image_manager._delete_single_image(self.image_sweeper, good_dir, self.image_id_1) assert_that(deleted is ret) assert_that(deleted is self.deleted) assert_that(marker_unlinked is self.marker_unlinked) @patch("host.hypervisor.esx.image_manager.EsxImageManager._get_datastore_type") @patch("host.hypervisor.esx.image_manager.EsxImageManager._get_mod_time") @patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rename") @patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rm_rf") def test_delete_single_image_no_marker_file( self, rm_rf, rename, get_mod_time, get_datastore_type): get_datastore_type.side_effect = self.patched_get_datastore_type get_mod_time.side_effect = self.patched_get_mod_time rename.side_effect = self.patched_rename rm_rf.side_effect = self.patched_rm_rf good_dir = os.path.join(self.test_dir, "image_" + self.image_id_2) deleted = self.image_manager._delete_single_image(self.image_sweeper, good_dir, self.image_id_2) assert_that(deleted is False) assert_that(self.deleted is False) @patch("host.hypervisor.esx.image_manager.EsxImageManager._read_marker_file") @patch("host.hypervisor.esx.image_manager.EsxImageManager._get_datastore_type") @patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rename") @patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rm_rf") def test_delete_single_image_no_timestamp_files( self, rm_rf, rename, get_datastore_type, read_marker_file): read_marker_file.side_effect = self.patched_read_marker_file get_datastore_type.side_effect = self.patched_get_datastore_type rename.side_effect = self.patched_rename rm_rf.side_effect = self.patched_rm_rf self.marker_file_content_time = 1000 good_dir = os.path.join(self.test_dir, "image_" + self.image_id_3) deleted = self.image_manager._delete_single_image(self.image_sweeper, good_dir, self.image_id_3) assert_that(deleted is True) assert_that(self.deleted is True) @parameterized.expand([ # marker time, mod time, mod time after rename, deleted (1061, 1000, True), (1060, 1000, False), (1000, 1000, False), (1000, 1001, False) ]) @patch("host.hypervisor.esx.image_manager.EsxImageManager._read_marker_file") @patch("host.hypervisor.esx.image_manager.EsxImageManager._get_datastore_type") @patch("host.hypervisor.esx.image_manager.EsxImageManager._get_mod_time") @patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rename") @patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_unlink") @patch("host.hypervisor.esx.image_manager.EsxImageManager._image_sweeper_rm_rf") def test_delete_single_image_no_timestamp_file( self, marker_file_content_time, renamed_timestamp_file_mod_time, deleted, rm_rf, unlink, rename, get_mod_time, get_datastore_type, read_marker_file): self.marker_file_content_time = marker_file_content_time self.renamed_timestamp_file_mod_time = renamed_timestamp_file_mod_time marker_unlinked = not deleted read_marker_file.side_effect = self.patched_read_marker_file get_datastore_type.side_effect = self.patched_get_datastore_type get_mod_time.side_effect = self.patched_get_mod_time rename.side_effect = self.patched_rename unlink.side_effect = self.patched_unlink rm_rf.side_effect = self.patched_rm_rf good_dir = os.path.join(self.test_dir, "image_" + self.image_id_4) ret = self.image_manager._delete_single_image(self.image_sweeper, good_dir, self.image_id_4) assert_that(deleted is ret) assert_that(deleted is self.deleted) assert_that(marker_unlinked is self.marker_unlinked) def patched_read_marker_file(self, filename): if not os.path.exists(filename): raise OSError return self.marker_file_content_time def patched_get_datastore_type(self, datastore_id): return DatastoreType.EXT3 def patched_get_mod_time(self, filename): try: os.path.getmtime(filename) except OSError as ex: if ex.errno == errno.ENOENT: return False, 0 else: raise ex # fix mod_time if filename.endswith( self.IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX): mod_time = self.renamed_timestamp_file_mod_time else: mod_time = self.timestamp_file_mod_time return True, mod_time def patched_rename(self, source, destination): if source.endswith(self.IMAGE_TIMESTAMP_FILENAME): shutil.move(source, destination) else: shutil.move(source, self.gc_dir) def patched_unlink(self, target): self.marker_unlinked = True def patched_rm_rf(self, target): self.deleted = True
def setUp(self, connect, update, creds): creds.return_value = ["username", "password"] self.vim_client = VimClient(auto_sync=False) self.ds_manager = MagicMock() services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(self.vim_client, self.ds_manager)
class TestEsxImageManager(unittest.TestCase): """Image Manager tests.""" # We can use even more unit test coverage of the image manager here @patch.object(VimClient, "acquire_credentials") @patch.object(VimClient, "update_cache") @patch("pysdk.connect.Connect") def setUp(self, connect, update, creds): creds.return_value = ["username", "password"] self.vim_client = VimClient(auto_sync=False) self.ds_manager = MagicMock() services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(self.vim_client, self.ds_manager) def tearDown(self): self.vim_client.disconnect(wait=True) @patch("os.path.isdir", return_value=False) @patch("os.makedirs", side_effect=OSError) def test_make_image_dir(self, _makedirs, _isdir): self.assertRaises( OSError, self.image_manager._make_image_dir, "ds", "fake_iid") _isdir.assert_called_once_with("/vmfs/volumes/ds/images/fa/fake_iid") self.assertEqual( _makedirs.call_count, EsxImageManager.NUM_MAKEDIRS_ATTEMPTS) for i in range(0, EsxImageManager.NUM_MAKEDIRS_ATTEMPTS): self.assertEqual(_makedirs.call_args_list[i][0], ("/vmfs/volumes/ds/images/fa/fake_iid",)) @patch( "host.hypervisor.esx.image_manager.EsxImageManager.reap_tmp_images") def test_periodic_reaper(self, mock_reap): """ Test that the we invoke the image reaper periodically """ image_manager = EsxImageManager(self.vim_client, self.ds_manager) image_manager.monitor_for_cleanup(reap_interval=0.1) self.assertFalse(image_manager._image_reaper is None) retry = 0 while mock_reap.call_count < 2 and retry < 10: time.sleep(0.1) retry += 1 image_manager.cleanup() assert_that(mock_reap.call_count, greater_than(1)) assert_that(retry, is_not(10), "reaper cleanup not called repeatedly") @patch("uuid.uuid4", return_value="fake_id") @patch("host.hypervisor.esx.vm_config.os_datastore_path") def test_reap_tmp_images(self, _os_datastore_path, _uuid): """ Test that stray images are found and deleted by the reaper """ def _fake_ds_folder(datastore, folder): return "%s__%s" % (datastore, folder) ds = MagicMock() ds.id = "dsid" ds.type = DatastoreType.EXT3 # In a random transient directory, set up a directory to act as the # tmp images folder and to contain a stray image folder with a file. tmpdir = file_util.mkdtemp(delete=True) tmp_images_folder = _fake_ds_folder(ds.id, TMP_IMAGE_FOLDER_NAME) tmp_images_dir = os.path.join(tmpdir, tmp_images_folder) tmp_image_dir = os.path.join(tmp_images_dir, "stray_image") os.mkdir(tmp_images_dir) os.mkdir(tmp_image_dir) (fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir) self.assertTrue(os.path.exists(path)) def _fake_os_datastore_path(datastore, folder): return os.path.join(tmpdir, _fake_ds_folder(datastore, folder)) _os_datastore_path.side_effect = _fake_os_datastore_path ds_manager = MagicMock() ds_manager.get_datastores.return_value = [ds] image_manager = EsxImageManager(self.vim_client, ds_manager) image_manager.reap_tmp_images() # verify stray image is deleted self.assertFalse(os.path.exists(path)) @patch("os.path.isdir") @patch("os.makedirs") def test_vmdk_mkdir_eexist(self, _makedirs, _isdir): eexist = OSError() eexist.errno = errno.EEXIST _makedirs.side_effect = eexist _isdir.side_effect = (False, # dest image dir missing True) # dest image dir is created self.image_manager._make_image_dir("ds", "fake_iid") _isdir.assert_called("/vmfs/volumes/ds/images/fa/fake_iid") @patch("pysdk.task.WaitForTask") @patch("uuid.uuid4", return_value="fake_id") @patch("os.path.exists") @patch("os.makedirs") @patch("shutil.copy") @patch("shutil.rmtree") @patch("shutil.move") @patch.object(EsxImageManager, "_manage_disk") @patch.object(EsxImageManager, "_get_datastore_type", return_value=DatastoreType.EXT3) @patch.object(EsxImageManager, "_check_image_repair", return_value=False) @patch.object(EsxImageManager, "check_and_validate_image", return_value=False) @patch.object(EsxImageManager, "_create_image_timestamp_file") @patch("host.hypervisor.esx.image_manager.FileBackedLock") def test_copy_image(self, _flock, _create_image_timestamp, check_image, _check_image_repair, _get_ds_type, _manage_disk, _mv_dir, _rmtree, _copy, _makedirs, _exists, _uuid, _wait_for_task): _exists.side_effect = (True, # dest image vmdk missing True) # source meta file present self.image_manager.copy_image("ds1", "foo", "ds2", "bar") os_path_prefix1 = '/vmfs/volumes/ds1/images' os_path_prefix2 = '/vmfs/volumes/ds2/images' os_tmp_path_prefix = '/vmfs/volumes/ds2/tmp_images' _copy.assert_called_once_with( '%s/fo/foo/foo.%s' % (os_path_prefix1, METADATA_FILE_EXT), '/vmfs/volumes/ds2/tmp_images/fake_id') ds_path_prefix1 = '[] ' + os_path_prefix1 ds_tmp_path_prefix = '[] ' + os_tmp_path_prefix expected_tmp_disk_ds_path = '%s/fake_id/%s.vmdk' % (ds_tmp_path_prefix, 'bar') _vd_spec = _manage_disk.call_args_list[0][1]['destSpec'] self.assertEqual("thin", _vd_spec.diskType) self.assertEqual("lsiLogic", _vd_spec.adapterType) copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task, sourceName='%s/fo/foo/foo.vmdk' % ds_path_prefix1, destName=expected_tmp_disk_ds_path, destSpec=_vd_spec) expected_vim_calls = [copy_call] self.assertEqual(expected_vim_calls, _manage_disk.call_args_list) _mv_dir.assert_called_once_with('/vmfs/volumes/ds2/tmp_images/fake_id', '%s/ba/bar' % os_path_prefix2) _create_image_timestamp.assert_called_once_with( "/vmfs/volumes/ds2/tmp_images/fake_id") @patch("pysdk.task.WaitForTask") @patch("uuid.uuid4", return_value="fake_id") @patch("os.path.exists") @patch("os.makedirs") @patch("shutil.copy") @patch.object(EsxImageManager, "_manage_disk") @patch.object(EsxImageManager, "_get_datastore_type", return_value=DatastoreType.EXT3) @patch.object(EsxImageManager, "check_image", return_value=False) @patch.object(EsxImageManager, "_create_image_timestamp_file") @patch("host.hypervisor.esx.image_manager.FileBackedLock") def test_create_tmp_image(self, _flock, _create_image_timestamp, check_image, _get_ds_type, _manage_disk, _copy, _makedirs, _exists, _uuid, _wait_for_task): # Common case is the same as the one covered by test_copy_image. # Check that things work when the src metadata file doesn't exist. _exists.side_effect = (False, True) ds_path_prefix1 = '[] /vmfs/volumes/ds1/images' expected_tmp_disk_ds_path = \ "[] /vmfs/volumes/ds2/tmp_images/fake_id/bar.vmdk" self.image_manager._create_tmp_image("ds1", "foo", "ds2", "bar") _flock.assert_called_once_with("/vmfs/volumes/ds2/tmp_images/fake_id", DatastoreType.EXT3) # Verify that we don't copy the metadata file. self.assertFalse(_copy.called) # Verify that we copy the disk correctly _vd_spec = _manage_disk.call_args_list[0][1]['destSpec'] self.assertEqual("thin", _vd_spec.diskType) self.assertEqual("lsiLogic", _vd_spec.adapterType) copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task, sourceName='%s/fo/foo/foo.vmdk' % ds_path_prefix1, destName=expected_tmp_disk_ds_path, destSpec=_vd_spec) expected_vim_calls = [copy_call] self.assertEqual(expected_vim_calls, _manage_disk.call_args_list) # check that we return an IO error if the copy of metadata fails. _copy.side_effect = IOError _exists.side_effect = (True, True) _manage_disk.reset_mock() _flock.reset_mock() self.assertRaises(IOError, self.image_manager._create_tmp_image, "ds1", "foo", "ds2", "bar") self.assertFalse(_manage_disk.called) _flock.assert_called_once_with("/vmfs/volumes/ds2/tmp_images/fake_id", DatastoreType.EXT3) _create_image_timestamp.assert_called_once_with( "/vmfs/volumes/ds2/tmp_images/fake_id") @patch("os.makedirs") @patch("shutil.rmtree") @patch("shutil.move") @patch.object(EsxImageManager, "_get_datastore_type", return_value=DatastoreType.EXT3) @patch.object(EsxImageManager, "_check_image_repair", return_value=True) @patch("host.hypervisor.esx.image_manager.FileBackedLock") @raises(DiskAlreadyExistException) def test_move_image(self, _flock, check_image, _get_ds_type, _mv_dir, _rmtree, _makedirs): # Common case is covered in test_copy_image. # check that if destination image directory exists we don't call move # and just bail after removing the tmp dir _rmtree.reset_mock() _mv_dir.reset_mock() expected_tmp_disk_folder = '/vmfs/volumes/ds2/tmp_images/bar' expected_rm_calls = [call(expected_tmp_disk_folder)] self.image_manager._move_image("foo", "ds1", expected_tmp_disk_folder) self.assertEqual(expected_rm_calls, _rmtree.call_args_list) _makedirs.assert_called_once_with('/vmfs/volumes/ds1/images/fo') _flock.assert_called_once_with('/vmfs/volumes/ds1/images/fo/foo', DatastoreType.EXT3, 3) @parameterized.expand([ (True, ), (False, ) ]) @patch("os.path.exists") @patch.object(EsxImageManager, "_get_datastore_type", return_value=DatastoreType.EXT3) @patch.object(EsxImageManager, "_create_image_timestamp_file") @patch.object(EsxImageManager, "_delete_renamed_image_timestamp_file") @patch("host.hypervisor.esx.image_manager.FileBackedLock") def test_validate_existing_image(self, create, _flock, _delete_renamed_timestamp_file, _create_timestamp_file, _get_ds_type, _path_exists): self._create_image_timestamp_file = create _path_exists.side_effect = self._local_os_path_exists _disk_folder = '/vmfs/volumes/ds1/images/fo/foo' self.image_manager._check_image_repair("foo", "ds1") if create: _create_timestamp_file.assert_called_once_with(_disk_folder) _delete_renamed_timestamp_file.assert_called_once() else: assert not _create_timestamp_file.called assert not _delete_renamed_timestamp_file.called def _local_os_path_exists(self, pathname): if not self._create_image_timestamp_file: return True if pathname.endswith(EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME): return False else: return True @patch.object(EsxImageManager, "_clean_gc_dir") @patch.object(EsxImageManager, "_gc_image_dir") @patch.object(EsxImageManager, "_lock_data_disk") @patch.object(EsxImageManager, "create_image_tombstone") @patch.object(EsxImageManager, "check_image_dir") def test_delete(self, check_image_dir, create_image_tombstone, lock_data_disk, gc_image_dir, clean_gc_dir): # Test successful delete check_image_dir.return_value = True self.image_manager.delete_image("ds1", "foo", 0, False) check_image_dir.assert_called_with("foo", "ds1") create_image_tombstone.assert_called_with("ds1", "foo") # Test successful delete with force option self.image_manager.delete_image("ds1", "foo", 0, True) check_image_dir.assert_called_with("foo", "ds1") create_image_tombstone.assert_called_with("ds1", "foo") lock_data_disk.assert_called_with("ds1", "foo") gc_image_dir.assert_called_with("ds1", "foo") clean_gc_dir.assert_called() # Test image not found check_image_dir.return_value = False self.assertRaises(ImageNotFoundException, self.image_manager.delete_image, "ds1", "foo", 0, False) @patch("host.hypervisor.esx.image_manager.os_vmdk_path") @patch("host.hypervisor.esx.image_manager.os_datastore_path") def test_gc_image_dir(self, dst_path, src_path): """ Test that we move the directory correctly to the GC location """ src_dir = file_util.mkdtemp(delete=True) dst_dir = file_util.mkdtemp(delete=True) src_path.return_value = os.path.join(src_dir, "test.vmdk") dst_path.return_value = dst_dir self.image_manager._gc_image_dir("ds1", "foo") uuid_dir = os.path.join(dst_dir, os.listdir(dst_dir)[0]) # Verify the src directory has been moved into the garbage dir. self.assertEqual(os.listdir(uuid_dir), [os.path.basename(src_dir)]) src_path.assert_called_once_with("ds1", "foo", IMAGE_FOLDER_NAME) dst_path.assert_called_once_with("ds1", GC_IMAGE_FOLDER) def test_image_path(self): image_path = "/vmfs/volumes/ds/images/tt/ttylinux/ttylinux.vmdk" ds = self.image_manager.get_datastore_id_from_path(image_path) image = self.image_manager.get_image_id_from_path(image_path) self.assertEqual(ds, "ds") self.assertEqual(image, "ttylinux") @patch("host.hypervisor.esx.image_manager.os_vmdk_flat_path") @patch("host.hypervisor.esx.image_manager.os.remove") def test_lock_data_disk(self, mock_rm, vmdk_flat_path): """ Test acquisition of the lock on the flat file. """ vmdk_flat_path.return_value = "fake_f_name" self.assertTrue(self.image_manager._lock_data_disk("ds1", "foo")) vmdk_flat_path.assert_called_once_with("ds1", "foo") mock_rm.side_effect = OSError self.assertFalse(self.image_manager._lock_data_disk("ds1", "foo")) @parameterized.expand([ ("CLOUD", "EAGER", ImageType.CLOUD, ImageReplication.EAGER), ("MANAGEMENT", "EAGER", ImageType.MANAGEMENT, ImageReplication.EAGER), ("CLOUD", "ON_DEMAND", ImageType.CLOUD, ImageReplication.ON_DEMAND), ("MANAGEMENT", "ON_DEMAND", ImageType.MANAGEMENT, ImageReplication.ON_DEMAND), ]) def test_image_type(self, type, replication, expected_type, expected_replication): self.ds_manager.image_datastores.return_value = "ds1" with patch("host.hypervisor.esx.image_manager.os_image_manifest_path" "") as manifest_path: tmpdir = file_util.mkdtemp(delete=True) tmpfile = os.path.join(tmpdir, "ds1.manifest") manifest_path.return_value = tmpfile with open(tmpfile, 'w+') as f: f.write('{"imageType":"%s","imageReplication":"%s"}' % ( type, replication)) type, replication = self.image_manager.get_image_manifest( "image_id") self.assertEqual(type, expected_type) self.assertEqual(replication, expected_replication) @patch.object(EsxImageManager, "_move_image") @patch.object(EsxImageManager, "check_image_dir", return_value=False) @patch.object(EsxImageManager, "_create_image_timestamp_file_from_ids") @patch("os.path.exists") def test_create_image(self, _exists, _create_timestamp, check_image_dir, move_image): # Happy path verify move is called with the right args. _exists.side_effect = ([True]) self.image_manager.create_image("ds1", "foo", "img_1") check_image_dir.assert_called_once_with("img_1", "ds1") move_image.assert_called_once_with('img_1', 'ds1', '/vmfs/volumes/ds1/foo') _create_timestamp.assert_called_once_with("ds1", "img_1") # Verify error if tmp image doesn't exist _exists.side_effect = ([False]) move_image.reset_mock() self.assertRaises(ImageNotFoundException, self.image_manager.create_image, "ds1", "foo", "img_1") self.assertFalse(move_image.called) # Verify error if destination image already exists. _exists.side_effect = ([True]) move_image.reset_mock() check_image_dir.return_value = True self.assertRaises(DiskAlreadyExistException, self.image_manager.create_image, "ds1", "foo", "img_1") self.assertFalse(move_image.called) @patch.object(EsxImageManager, "create_image") @patch.object(EsxImageManager, "_manage_disk") @patch("os.path.exists", return_value=True) def test_create_image_with_vm_disk(self, _exists, _manage_disk, _create_image): vm_disk_path = "/vmfs/volumes/dsname/vms/ab/cd.vmdk" self.image_manager.create_image_with_vm_disk( "ds1", "foo", "img_1", vm_disk_path) # Verify that we copy the disk correctly expected_tmp_disk_ds_path = \ "[] /vmfs/volumes/ds1/foo/img_1.vmdk" _vd_spec = _manage_disk.call_args_list[0][1]['destSpec'] self.assertEqual("thin", _vd_spec.diskType) self.assertEqual("lsiLogic", _vd_spec.adapterType) copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task, sourceName='[] %s' % vm_disk_path, destName=expected_tmp_disk_ds_path, destSpec=_vd_spec) expected_vim_calls = [copy_call] self.assertEqual(expected_vim_calls, _manage_disk.call_args_list) _create_image.assert_called_once_with("ds1", "foo", "img_1") @patch("shutil.rmtree") @patch("os.path.exists") def test_delete_tmp_dir(self, _exists, _rmtree): self.image_manager.delete_tmp_dir("ds1", "foo") _exists.assert_called_once("/vmfs/volumes/ds1/foo") _rmtree.assert_called_once("/vmfs/volumes/ds1/foo") _exists.reset_mock() _exists.return_value = False _rmtree.reset_mock() self.assertRaises(DirectoryNotFound, self.image_manager.delete_tmp_dir, "ds1", "foo") _exists.assert_called_once("/vmfs/volumes/ds1/foo") self.assertFalse(_rmtree.called)
class TestEsxImageManager(unittest.TestCase): """Image Manager tests.""" # We can use even more unit test coverage of the image manager here @patch.object(VimClient, "acquire_credentials") @patch.object(VimClient, "update_cache") @patch("pysdk.connect.Connect") def setUp(self, connect, update, creds): creds.return_value = ["username", "password"] self.vim_client = VimClient(auto_sync=False) self.ds_manager = MagicMock() services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(self.vim_client, self.ds_manager) def tearDown(self): self.vim_client.disconnect(wait=True) @patch("os.path.isdir", return_value=False) @patch("os.makedirs", side_effect=OSError) def test_make_image_dir(self, _makedirs, _isdir): path = "/vmfs/volumes/ds/image_fake_iid" self.assertRaises( OSError, self.image_manager._make_image_dir, "ds", "fake_iid") _isdir.assert_called_once_with(path) self.assertEqual( _makedirs.call_count, EsxImageManager.NUM_MAKEDIRS_ATTEMPTS) for i in range(0, EsxImageManager.NUM_MAKEDIRS_ATTEMPTS): self.assertEqual(_makedirs.call_args_list[i][0], (path,)) @patch( "host.hypervisor.esx.image_manager.EsxImageManager.reap_tmp_images") def test_periodic_reaper(self, mock_reap): """ Test that the we invoke the image reaper periodically """ image_manager = EsxImageManager(self.vim_client, self.ds_manager) image_manager.monitor_for_cleanup(reap_interval=0.1) self.assertFalse(image_manager._image_reaper is None) retry = 0 while mock_reap.call_count < 2 and retry < 10: time.sleep(0.1) retry += 1 image_manager.cleanup() assert_that(mock_reap.call_count, greater_than(1)) assert_that(retry, is_not(10), "reaper cleanup not called repeatedly") @parameterized.expand([ (True, ), (False, ) ]) @patch("uuid.uuid4", return_value="fake_id") @patch("host.hypervisor.esx.vm_config.os_datastore_root") def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_root, _uuid): """ Test that stray images are found and deleted by the reaper """ ds = MagicMock() ds.id = "dsid" ds.type = DatastoreType.EXT3 # In a random transient directory, set up a directory to act as the # tmp images folder and to contain a stray image folder with a file. tmpdir = file_util.mkdtemp(delete=True) tmp_ds_dir = os.path.join(tmpdir, ds.id) os.mkdir(tmp_ds_dir) tmp_image_dir = os.path.join(tmp_ds_dir, compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image")) os.mkdir(tmp_image_dir) (fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir) self.assertTrue(os.path.exists(path)) def _fake_os_datastore_root(datastore): return os.path.join(tmpdir, datastore) _os_datastore_root.side_effect = _fake_os_datastore_root ds_manager = MagicMock() ds_manager.get_datastores.return_value = [ds] image_manager = EsxImageManager(self.vim_client, ds_manager) if not _allow_grace_period: image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0 time.sleep(0.1) image_manager.reap_tmp_images() if _allow_grace_period: # verify stray image is not deleted due to grace period self.assertTrue(os.path.exists(path)) else: # verify stray image is deleted self.assertFalse(os.path.exists(path)) @patch("os.path.isdir") @patch("os.makedirs") def test_vmdk_mkdir_eexist(self, _makedirs, _isdir): eexist = OSError() eexist.errno = errno.EEXIST _makedirs.side_effect = eexist _isdir.side_effect = (False, # dest image dir missing True) # dest image dir is created self.image_manager._make_image_dir("ds", "fake_iid") _isdir.assert_called("/vmfs/volumes/ds/image_fake_iid") @patch("pysdk.task.WaitForTask") @patch("uuid.uuid4", return_value="fake_id") @patch("os.path.exists") @patch("shutil.copy") @patch.object(VimClient, "move_file") @patch.object(EsxImageManager, "_manage_disk") @patch.object(EsxImageManager, "_get_datastore_type", return_value=DatastoreType.EXT3) @patch.object(EsxImageManager, "_check_image_repair", return_value=False) @patch.object(EsxImageManager, "check_and_validate_image", return_value=False) @patch.object(EsxImageManager, "_create_image_timestamp_file") @patch("host.hypervisor.esx.image_manager.FileBackedLock") def test_copy_image(self, _flock, _create_image_timestamp, check_image, _check_image_repair, _get_ds_type, _manage_disk, _mv_dir, _copy, _exists, _uuid, _wait_for_task): _exists.side_effect = (True, # tmp_dir exists True, # dest image vmdk missing True) # source meta file present self.image_manager.copy_image("ds1", "foo", "ds2", "bar") os_path_prefix1 = '/vmfs/volumes/ds1' os_path_prefix2 = '/vmfs/volumes/ds2' ds_tmp_path_prefix = '[] /vmfs/volumes/ds2' assert_that(_copy.call_count, equal_to(1)) _copy.assert_has_calls([ call('%s/image_foo/foo.%s' % (os_path_prefix1, METADATA_FILE_EXT), '/vmfs/volumes/ds2/tmp_image_fake_id/bar.%s' % METADATA_FILE_EXT), ]) ds_path_prefix1 = '[] ' + os_path_prefix1 expected_tmp_disk_ds_path = '%s/tmp_image_fake_id/bar.vmdk' % (ds_tmp_path_prefix) _vd_spec = _manage_disk.call_args_list[0][1]['destSpec'] self.assertEqual("thin", _vd_spec.diskType) self.assertEqual("lsiLogic", _vd_spec.adapterType) copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task, sourceName='%s/image_foo/foo.vmdk' % ds_path_prefix1, destName=expected_tmp_disk_ds_path, destSpec=_vd_spec) expected_vim_calls = [copy_call] self.assertEqual(expected_vim_calls, _manage_disk.call_args_list) _mv_dir.assert_called_once_with('/vmfs/volumes/ds2/tmp_image_fake_id', '%s/image_bar' % os_path_prefix2) _create_image_timestamp.assert_called_once_with( "/vmfs/volumes/ds2/tmp_image_fake_id") @patch("pysdk.task.WaitForTask") @patch("uuid.uuid4", return_value="fake_id") @patch("os.path.exists") @patch("os.makedirs") @patch("shutil.copy") @patch.object(EsxImageManager, "_manage_disk") @patch.object(EsxImageManager, "_get_datastore_type", return_value=DatastoreType.EXT3) @patch.object(EsxImageManager, "check_image", return_value=False) @patch.object(EsxImageManager, "_create_image_timestamp_file") def test_create_tmp_image(self, _create_image_timestamp, check_image, _get_ds_type, _manage_disk, _copy, _makedirs, _exists, _uuid, _wait_for_task): # Common case is the same as the one covered by test_copy_image. # Check that things work when the src metadata file doesn't exist. _exists.side_effect = (False, False, True) ds_path_prefix1 = '[] /vmfs/volumes/ds1' expected_tmp_disk_ds_path = \ "[] /vmfs/volumes/ds2/tmp_image_fake_id/bar.vmdk" self.image_manager._copy_to_tmp_image("ds1", "foo", "ds2", "bar") # Verify that we don't copy the metadata file. self.assertFalse(_copy.called) # Verify that we copy the disk correctly _vd_spec = _manage_disk.call_args_list[0][1]['destSpec'] self.assertEqual("thin", _vd_spec.diskType) self.assertEqual("lsiLogic", _vd_spec.adapterType) copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task, sourceName='%s/image_foo/foo.vmdk' % ds_path_prefix1, destName=expected_tmp_disk_ds_path, destSpec=_vd_spec) expected_vim_calls = [copy_call] self.assertEqual(expected_vim_calls, _manage_disk.call_args_list) # check that we return an IO error if the copy of metadata fails. _copy.side_effect = IOError _exists.side_effect = (True, True) _manage_disk.reset_mock() self.assertRaises(IOError, self.image_manager._copy_to_tmp_image, "ds1", "foo", "ds2", "bar") self.assertFalse(_manage_disk.called) _create_image_timestamp.assert_called_once_with( "/vmfs/volumes/ds2/tmp_image_fake_id") @patch.object(VimClient, "delete_file") @patch("os.path.exists", return_value=True) @patch("os.makedirs") @patch("shutil.rmtree") @patch("shutil.move") @patch.object(EsxImageManager, "_get_datastore_type", return_value=DatastoreType.EXT3) @patch.object(EsxImageManager, "_check_image_repair", return_value=True) @patch("host.hypervisor.esx.image_manager.FileBackedLock") @raises(DiskAlreadyExistException) def test_move_image(self, _flock, check_image, _get_ds_type, _mv_dir, _rmtree, _makedirs, _exists, _delete_file): # Common case is covered in test_copy_image. # check that if destination image directory exists we don't call move # and just bail after removing the tmp dir _rmtree.reset_mock() _mv_dir.reset_mock() expected_tmp_disk_folder = '/vmfs/volumes/ds2/tmp_images/bar' expected_rm_calls = [call(expected_tmp_disk_folder)] self.image_manager._move_image("foo", "ds1", expected_tmp_disk_folder) self.assertEqual(expected_rm_calls, _rmtree.call_args_list) _makedirs.assert_called_once_with('/vmfs/volumes/ds1/images/fo') _flock.assert_called_once_with('/vmfs/volumes/ds1/image_foo', DatastoreType.EXT3, 3) _delete_file.assert_called_once_with('/vmfs/volumes/ds1/image_foo') @parameterized.expand([ (True, ), (False, ) ]) @patch("os.path.exists") @patch.object(EsxImageManager, "_get_datastore_type", return_value=DatastoreType.EXT3) @patch.object(EsxImageManager, "_create_image_timestamp_file") @patch.object(EsxImageManager, "_delete_renamed_image_timestamp_file") @patch("host.hypervisor.esx.image_manager.FileBackedLock") def test_validate_existing_image(self, create, _flock, _delete_renamed_timestamp_file, _create_timestamp_file, _get_ds_type, _path_exists): self._create_image_timestamp_file = create _path_exists.side_effect = self._local_os_path_exists _disk_folder = '/vmfs/volumes/ds1/image_foo' self.image_manager._check_image_repair("foo", "ds1") if create: _create_timestamp_file.assert_called_once_with(_disk_folder) _delete_renamed_timestamp_file.assert_called_once() else: assert not _create_timestamp_file.called assert not _delete_renamed_timestamp_file.called def _local_os_path_exists(self, pathname): if not self._create_image_timestamp_file: return True if pathname.endswith(EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME): return False else: return True def test_image_path(self): image_path = "/vmfs/volumes/ds/image_ttylinux/ttylinux.vmdk" ds = self.image_manager.get_datastore_id_from_path(image_path) image = self.image_manager.get_image_id_from_path(image_path) self.assertEqual(ds, "ds") self.assertEqual(image, "ttylinux") @patch.object(EsxImageManager, "_get_datastore_type") def test_create_image(self, _get_ds_type): image_id = "image_id" datastore_id = "ds1" _get_ds_type.side_effect = (DatastoreType.LOCAL_VMFS, DatastoreType.VSAN) tmp_image_path = self.image_manager.create_image(image_id, datastore_id) prefix = "[] /vmfs/volumes/%s/tmp_image_" % datastore_id self.assertTrue(tmp_image_path.startswith(prefix)) tmp_image_path = self.image_manager.create_image(image_id, datastore_id) prefix = "[] /vmfs/volumes/%s/image_%s/tmp_image_" % (datastore_id, image_id) self.assertTrue(tmp_image_path.startswith(prefix)) @patch.object(EsxImageManager, "_move_image") @patch.object(EsxImageManager, "_create_image_timestamp_file_from_ids") @patch("os.path.exists") def test_finalize_image(self, _exists, _create_timestamp, move_image): # Happy path verify move is called with the right args. _exists.side_effect = ([True]) self.image_manager.finalize_image("ds1", "[] /vmfs/volumes/ds1/foo", "img_1") move_image.assert_called_once_with('img_1', 'ds1', '/vmfs/volumes/ds1/foo') _create_timestamp.assert_called_once_with("ds1", "img_1") @patch.object(EsxImageManager, "finalize_image") @patch.object(EsxImageManager, "_manage_disk") @patch("os.path.exists", return_value=True) def test_create_image_with_vm_disk(self, _exists, _manage_disk, _create_image): vm_disk_path = "/vmfs/volumes/dsname/vms/ab/cd.vmdk" self.image_manager.create_image_with_vm_disk( "ds1", "[] /vmfs/volumes/ds1/foo", "img_1", vm_disk_path) # Verify that we copy the disk correctly expected_tmp_disk_ds_path = \ "[] /vmfs/volumes/ds1/foo/img_1.vmdk" _vd_spec = _manage_disk.call_args_list[0][1]['destSpec'] self.assertEqual("thin", _vd_spec.diskType) self.assertEqual("lsiLogic", _vd_spec.adapterType) copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task, sourceName='[] %s' % vm_disk_path, destName=expected_tmp_disk_ds_path, destSpec=_vd_spec) expected_vim_calls = [copy_call] self.assertEqual(expected_vim_calls, _manage_disk.call_args_list) _create_image.assert_called_once_with("ds1", "[] /vmfs/volumes/ds1/foo", "img_1") @patch("shutil.rmtree") @patch("os.path.exists") def test_delete_tmp_dir(self, _exists, _rmtree): self.image_manager.delete_tmp_dir("ds1", "foo") _exists.assert_called_once("/vmfs/volumes/ds1/foo") _rmtree.assert_called_once("/vmfs/volumes/ds1/foo") _exists.reset_mock() _exists.return_value = False _rmtree.reset_mock() self.assertRaises(DirectoryNotFound, self.image_manager.delete_tmp_dir, "ds1", "foo") _exists.assert_called_once("/vmfs/volumes/ds1/foo") self.assertFalse(_rmtree.called) def test_image_size(self): self.ds_manager.image_datastores.return_value = ["ds1", "ds2"] with patch("host.hypervisor.esx.image_manager.os_vmdk_flat_path" "") as image_path: tmpdir = file_util.mkdtemp(delete=True) image_path.return_value = tmpdir size = self.image_manager.image_size("image_id") self.assertTrue(size > 0) def test_image_size_not_exist(self): self.ds_manager.image_datastores.return_value = ["ds1", "ds2"] self.assertRaises(NoSuchResourceException, self.image_manager.image_size, "image_id")
class EsxHypervisor(object): """Manage ESX Hypervisor.""" def __init__(self, agent_config): self.logger = logging.getLogger(__name__) # If VimClient's housekeeping thread failed to update its own cache, # call errback to commit suicide. Watchdog will bring up the agent # again. self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout, errback=lambda: suicide()) atexit.register(lambda client: client.disconnect(), self.vim_client) self._uuid = self.vim_client.host_uuid self.set_memory_overcommit(agent_config.memory_overcommit) self.datastore_manager = EsxDatastoreManager( self, agent_config.datastores, agent_config.image_datastores) # datastore manager needs to update the cache when there is a change. self.vim_client.add_update_listener(self.datastore_manager) self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager) self.disk_manager = EsxDiskManager(self.vim_client, self.datastore_manager) self.image_manager = EsxImageManager(self.vim_client, self.datastore_manager) self.network_manager = EsxNetworkManager(self.vim_client, agent_config.networks) self.system = EsxSystem(self.vim_client) self.image_manager.monitor_for_cleanup() self.image_transferer = HttpNfcTransferer( self.vim_client, self.datastore_manager.image_datastores()) atexit.register(self.image_manager.cleanup) @property def uuid(self): return self._uuid def check_image(self, image_id, datastore_id): return self.image_manager.check_image( image_id, self.datastore_manager.datastore_name(datastore_id)) def acquire_vim_ticket(self): return self.vim_client.acquire_clone_ticket() def acquire_cgi_ticket(self, url, op): return self.vim_client.acquire_cgi_ticket(url, op) def add_update_listener(self, listener): self.vim_client.add_update_listener(listener) def remove_update_listener(self, listener): self.vim_client.remove_update_listener(listener) def transfer_image(self, source_image_id, source_datastore, destination_image_id, destination_datastore, host, port): return self.image_transferer.send_image_to_host( source_image_id, source_datastore, destination_image_id, destination_datastore, host, port) def receive_image(self, image_id, datastore, imported_vm_name, metadata, manifest): self.image_manager.receive_image(image_id, datastore, imported_vm_name, metadata, manifest) def set_memory_overcommit(self, memory_overcommit): # Enable/Disable large page support. If this host is removed # from the deployment, large page support will need to be # explicitly updated by the user. disable_large_pages = memory_overcommit > 1.0 self.vim_client.set_large_page_support(disable=disable_large_pages)
class ImageSweeperTouchTimestampTestCase(unittest.TestCase): DATASTORE_ID = "DS01" BASE_TEMP_DIR = "image_sweeper" IMAGE_TIMESTAMP_FILENAME = EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME def setUp(self): self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.vm_manager = MagicMock() self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID) self.image_sweeper._task_runner = MagicMock() self.image_sweeper._task_runner.is_stopped.return_value = False self.delete_count = 0 # Create various image directories and empty vmdks dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "image_") self.dir0 = dir0 # Image dir with correct timestamp file image_id_1 = str(uuid.uuid4()) dir1 = self.create_dir(image_id_1) open(os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close() # Image dir without the correct timestamp file image_id_2 = str(uuid.uuid4()) dir2 = self.create_dir(image_id_2) # Image dir with correct timestamp file # and with tombstone file image_id_3 = str(uuid.uuid4()) dir3 = self.create_dir(image_id_3) open(os.path.join( dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close() self.image_ids = ["", image_id_1, image_id_2, image_id_3] self.image_dirs = ["", dir1, dir2, dir3] def tearDown(self): shutil.rmtree(self.test_dir, True) @parameterized.expand([ # timestamp_exists (True, ), (False, ) ]) # The os_vmdk_path method is defined in vm_config.py # but it is imported in esx/image_manager.py, that is # the instance we need to patch @patch("host.hypervisor.esx.image_manager.os_vmdk_path") def test_touch_timestamp_file(self, timestamp_exists, os_vmdk_path): if not timestamp_exists: image_index = 2 exception_class = type(OSError()) else: image_index = 1 exception_class = None image_id = self.image_ids[image_index] image_dir = self.image_dirs[image_index] os_vmdk_path.side_effect = self.patched_os_vmdk_path timestamp_filename_path = os.path.join(image_dir, self.IMAGE_TIMESTAMP_FILENAME) pre_mod_time = 0 if timestamp_exists: # save mod time on the image timestamp file pre_mod_time = os.path.getmtime(timestamp_filename_path) try: time.sleep(1) self.image_manager.touch_image_timestamp(self.DATASTORE_ID, image_id) assert_that(exception_class is None) # check new timestamp post_mod_time = os.path.getmtime(timestamp_filename_path) assert_that((post_mod_time > pre_mod_time) is True) except Exception as ex: assert_that((type(ex) == exception_class) is True) def patched_os_vmdk_path(self, datastore, disk_id, folder): folder = self.dir0 ret = os_vmdk_path(datastore, disk_id, folder) return ret def create_dir(self, image_id): dirname = vm_config.compond_path_join(self.dir0, image_id) os.makedirs(dirname) return dirname
def setUp(self): self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) self.gc_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.vm_manager = MagicMock() self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID) self.deleted = False self.marker_unlinked = False # Create various image directories and empty vmdks image_id_1 = str(uuid.uuid4()) image_id_2 = str(uuid.uuid4()) image_id_3 = str(uuid.uuid4()) image_id_4 = str(uuid.uuid4()) self.image_id_1 = image_id_1 self.image_id_2 = image_id_2 self.image_id_3 = image_id_3 self.image_id_4 = image_id_4 dir1 = os.path.join(self.test_dir, "image_" + image_id_1) os.makedirs(dir1) dir2 = os.path.join(self.test_dir, "image_" + image_id_2) os.makedirs(dir2) dir3 = os.path.join(self.test_dir, "image_" + image_id_3) os.makedirs(dir3) dir4 = os.path.join(self.test_dir, "image_" + image_id_4) os.makedirs(dir4) self.marker_file_content_time = 0 self.timestamp_file_mod_time = 0 self.renamed_timestamp_file_mod_time = 0 # Create a good image vmdk under image_id_1, # also create a valid image marker file # and a valid timestamp file vmdk_filename = image_id_1 + ".vmdk" open(os.path.join(dir1, vmdk_filename), 'w').close() timestamp_filename = os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME) open(timestamp_filename, 'w').close() marker_filename = os.path.join(dir1, self.IMAGE_MARKER_FILENAME) open(marker_filename, 'w').close() # Create a good image vmdk under image_id_2, # create timestamp but no image marker file, vmdk_filename = image_id_2 + ".vmdk" open(os.path.join(dir2, vmdk_filename), 'w').close() timestamp_filename = os.path.join(dir2, self.IMAGE_TIMESTAMP_FILENAME) open(timestamp_filename, 'w').close() # Create a good image vmdk under image_id_3, # create image_marker file but no timestamp # and no renamed timestamp file vmdk_filename = image_id_3 + ".vmdk" open(os.path.join(dir3, vmdk_filename), 'w').close() marker_filename = os.path.join(dir3, self.IMAGE_MARKER_FILENAME) open(marker_filename, 'w').close() # Create a good image vmdk under image_id_4, # create image_marker file, renamed timestamp file # but no timestamp file vmdk_filename = image_id_4 + ".vmdk" open(os.path.join(dir4, vmdk_filename), 'w').close() marker_filename = os.path.join(dir4, self.IMAGE_MARKER_FILENAME) open(marker_filename, 'w').close() timestamp_filename = os.path.join(dir4, self.IMAGE_TIMESTAMP_FILENAME) renamed_timestamp_filename = timestamp_filename + self.IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX open(renamed_timestamp_filename, 'w').close()
class TestEsxImageManager(unittest.TestCase): """Image Manager tests.""" # We can use even more unit test coverage of the image manager here @patch.object(VimClient, "acquire_credentials") @patch.object(VimClient, "update_cache") @patch("pysdk.connect.Connect") def setUp(self, connect, update, creds): creds.return_value = ["username", "password"] self.vim_client = VimClient(auto_sync=False) self.ds_manager = MagicMock() services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(self.vim_client, self.ds_manager) def tearDown(self): self.vim_client.disconnect(wait=True) @patch("os.path.isdir", return_value=False) @patch("os.makedirs", side_effect=OSError) def test_make_image_dir(self, _makedirs, _isdir): self.assertRaises( OSError, self.image_manager._make_image_dir, "ds", "fake_iid") _isdir.assert_called_once_with("/vmfs/volumes/ds/images/fa/fake_iid") self.assertEqual( _makedirs.call_count, EsxImageManager.NUM_MAKEDIRS_ATTEMPTS) for i in range(0, EsxImageManager.NUM_MAKEDIRS_ATTEMPTS): self.assertEqual(_makedirs.call_args_list[i][0], ("/vmfs/volumes/ds/images/fa/fake_iid",)) @patch( "host.hypervisor.esx.image_manager.EsxImageManager.reap_tmp_images") def test_periodic_reaper(self, mock_reap): """ Test that the we invoke the image reaper periodically """ image_manager = EsxImageManager(self.vim_client, self.ds_manager) image_manager.monitor_for_cleanup(reap_interval=0.1) self.assertFalse(image_manager._image_reaper is None) retry = 0 while mock_reap.call_count < 2 and retry < 10: time.sleep(0.1) retry += 1 image_manager.cleanup() assert_that(mock_reap.call_count, greater_than(1)) assert_that(retry, is_not(10), "reaper cleanup not called repeatedly") @parameterized.expand([ (True, ), (False, ) ]) @patch("uuid.uuid4", return_value="fake_id") @patch("host.hypervisor.esx.vm_config.os_datastore_path") def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_path, _uuid): """ Test that stray images are found and deleted by the reaper """ def _fake_ds_folder(datastore, folder): return "%s__%s" % (datastore, folder) ds = MagicMock() ds.id = "dsid" ds.type = DatastoreType.EXT3 # In a random transient directory, set up a directory to act as the # tmp images folder and to contain a stray image folder with a file. tmpdir = file_util.mkdtemp(delete=True) tmp_images_folder = _fake_ds_folder(ds.id, TMP_IMAGE_FOLDER_NAME) tmp_images_dir = os.path.join(tmpdir, tmp_images_folder) tmp_image_dir = os.path.join(tmp_images_dir, "stray_image") os.mkdir(tmp_images_dir) os.mkdir(tmp_image_dir) (fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir) self.assertTrue(os.path.exists(path)) def _fake_os_datastore_path(datastore, folder): return os.path.join(tmpdir, _fake_ds_folder(datastore, folder)) _os_datastore_path.side_effect = _fake_os_datastore_path ds_manager = MagicMock() ds_manager.get_datastores.return_value = [ds] image_manager = EsxImageManager(self.vim_client, ds_manager) if not _allow_grace_period: image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0 time.sleep(0.1) image_manager.reap_tmp_images() if _allow_grace_period: # verify stray image is not deleted due to grace period self.assertTrue(os.path.exists(path)) else: # verify stray image is deleted self.assertFalse(os.path.exists(path)) @patch("os.path.isdir") @patch("os.makedirs") def test_vmdk_mkdir_eexist(self, _makedirs, _isdir): eexist = OSError() eexist.errno = errno.EEXIST _makedirs.side_effect = eexist _isdir.side_effect = (False, # dest image dir missing True) # dest image dir is created self.image_manager._make_image_dir("ds", "fake_iid") _isdir.assert_called("/vmfs/volumes/ds/images/fa/fake_iid") @patch("pysdk.task.WaitForTask") @patch("uuid.uuid4", return_value="fake_id") @patch("os.path.exists") @patch("os.makedirs") @patch("shutil.copy") @patch("shutil.rmtree") @patch("shutil.move") @patch.object(EsxImageManager, "_manage_disk") @patch.object(EsxImageManager, "_get_datastore_type", return_value=DatastoreType.EXT3) @patch.object(EsxImageManager, "_check_image_repair", return_value=False) @patch.object(EsxImageManager, "check_and_validate_image", return_value=False) @patch.object(EsxImageManager, "_create_image_timestamp_file") @patch("host.hypervisor.esx.image_manager.FileBackedLock") def test_copy_image(self, _flock, _create_image_timestamp, check_image, _check_image_repair, _get_ds_type, _manage_disk, _mv_dir, _rmtree, _copy, _makedirs, _exists, _uuid, _wait_for_task): _exists.side_effect = (True, # dest image vmdk missing True, # source meta file present True) # source manifest file present self.image_manager.copy_image("ds1", "foo", "ds2", "bar") os_path_prefix1 = '/vmfs/volumes/ds1/images' os_path_prefix2 = '/vmfs/volumes/ds2/images' os_tmp_path_prefix = '/vmfs/volumes/ds2/tmp_images' assert_that(_copy.call_count, equal_to(2)) _copy.assert_has_calls([ call('%s/fo/foo/foo.%s' % (os_path_prefix1, METADATA_FILE_EXT), '/vmfs/volumes/ds2/tmp_images/fake_id/bar.%s' % METADATA_FILE_EXT), call('%s/fo/foo/foo.%s' % (os_path_prefix1, MANIFEST_FILE_EXT), '/vmfs/volumes/ds2/tmp_images/fake_id/bar.%s' % MANIFEST_FILE_EXT), ]) ds_path_prefix1 = '[] ' + os_path_prefix1 ds_tmp_path_prefix = '[] ' + os_tmp_path_prefix expected_tmp_disk_ds_path = '%s/fake_id/%s.vmdk' % (ds_tmp_path_prefix, 'bar') _vd_spec = _manage_disk.call_args_list[0][1]['destSpec'] self.assertEqual("thin", _vd_spec.diskType) self.assertEqual("lsiLogic", _vd_spec.adapterType) copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task, sourceName='%s/fo/foo/foo.vmdk' % ds_path_prefix1, destName=expected_tmp_disk_ds_path, destSpec=_vd_spec) expected_vim_calls = [copy_call] self.assertEqual(expected_vim_calls, _manage_disk.call_args_list) _mv_dir.assert_called_once_with('/vmfs/volumes/ds2/tmp_images/fake_id', '%s/ba/bar' % os_path_prefix2) _create_image_timestamp.assert_called_once_with( "/vmfs/volumes/ds2/tmp_images/fake_id") @patch("pysdk.task.WaitForTask") @patch("uuid.uuid4", return_value="fake_id") @patch("os.path.exists") @patch("os.makedirs") @patch("shutil.copy") @patch.object(EsxImageManager, "_manage_disk") @patch.object(EsxImageManager, "_get_datastore_type", return_value=DatastoreType.EXT3) @patch.object(EsxImageManager, "check_image", return_value=False) @patch.object(EsxImageManager, "_create_image_timestamp_file") @patch("host.hypervisor.esx.image_manager.FileBackedLock") def test_create_tmp_image(self, _flock, _create_image_timestamp, check_image, _get_ds_type, _manage_disk, _copy, _makedirs, _exists, _uuid, _wait_for_task): # Common case is the same as the one covered by test_copy_image. # Check that things work when the src metadata file doesn't exist. _exists.side_effect = (False, False, True) ds_path_prefix1 = '[] /vmfs/volumes/ds1/images' expected_tmp_disk_ds_path = \ "[] /vmfs/volumes/ds2/tmp_images/fake_id/bar.vmdk" self.image_manager._create_tmp_image("ds1", "foo", "ds2", "bar") _flock.assert_called_once_with("/vmfs/volumes/ds2/tmp_images/fake_id", DatastoreType.EXT3) # Verify that we don't copy the metadata file. self.assertFalse(_copy.called) # Verify that we copy the disk correctly _vd_spec = _manage_disk.call_args_list[0][1]['destSpec'] self.assertEqual("thin", _vd_spec.diskType) self.assertEqual("lsiLogic", _vd_spec.adapterType) copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task, sourceName='%s/fo/foo/foo.vmdk' % ds_path_prefix1, destName=expected_tmp_disk_ds_path, destSpec=_vd_spec) expected_vim_calls = [copy_call] self.assertEqual(expected_vim_calls, _manage_disk.call_args_list) # check that we return an IO error if the copy of metadata fails. _copy.side_effect = IOError _exists.side_effect = (True, True) _manage_disk.reset_mock() _flock.reset_mock() self.assertRaises(IOError, self.image_manager._create_tmp_image, "ds1", "foo", "ds2", "bar") self.assertFalse(_manage_disk.called) _flock.assert_called_once_with("/vmfs/volumes/ds2/tmp_images/fake_id", DatastoreType.EXT3) _create_image_timestamp.assert_called_once_with( "/vmfs/volumes/ds2/tmp_images/fake_id") @patch("os.makedirs") @patch("shutil.rmtree") @patch("shutil.move") @patch.object(EsxImageManager, "_get_datastore_type", return_value=DatastoreType.EXT3) @patch.object(EsxImageManager, "_check_image_repair", return_value=True) @patch("host.hypervisor.esx.image_manager.FileBackedLock") @raises(DiskAlreadyExistException) def test_move_image(self, _flock, check_image, _get_ds_type, _mv_dir, _rmtree, _makedirs): # Common case is covered in test_copy_image. # check that if destination image directory exists we don't call move # and just bail after removing the tmp dir _rmtree.reset_mock() _mv_dir.reset_mock() expected_tmp_disk_folder = '/vmfs/volumes/ds2/tmp_images/bar' expected_rm_calls = [call(expected_tmp_disk_folder)] self.image_manager._move_image("foo", "ds1", expected_tmp_disk_folder) self.assertEqual(expected_rm_calls, _rmtree.call_args_list) _makedirs.assert_called_once_with('/vmfs/volumes/ds1/images/fo') _flock.assert_called_once_with('/vmfs/volumes/ds1/images/fo/foo', DatastoreType.EXT3, 3) @parameterized.expand([ (True, ), (False, ) ]) @patch("os.path.exists") @patch.object(EsxImageManager, "_get_datastore_type", return_value=DatastoreType.EXT3) @patch.object(EsxImageManager, "_create_image_timestamp_file") @patch.object(EsxImageManager, "_delete_renamed_image_timestamp_file") @patch("host.hypervisor.esx.image_manager.FileBackedLock") def test_validate_existing_image(self, create, _flock, _delete_renamed_timestamp_file, _create_timestamp_file, _get_ds_type, _path_exists): self._create_image_timestamp_file = create _path_exists.side_effect = self._local_os_path_exists _disk_folder = '/vmfs/volumes/ds1/images/fo/foo' self.image_manager._check_image_repair("foo", "ds1") if create: _create_timestamp_file.assert_called_once_with(_disk_folder) _delete_renamed_timestamp_file.assert_called_once() else: assert not _create_timestamp_file.called assert not _delete_renamed_timestamp_file.called def _local_os_path_exists(self, pathname): if not self._create_image_timestamp_file: return True if pathname.endswith(EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME): return False else: return True @patch.object(EsxImageManager, "_clean_gc_dir") @patch.object(EsxImageManager, "_gc_image_dir") @patch.object(EsxImageManager, "_lock_data_disk") @patch.object(EsxImageManager, "create_image_tombstone") @patch.object(EsxImageManager, "check_image_dir") def test_delete(self, check_image_dir, create_image_tombstone, lock_data_disk, gc_image_dir, clean_gc_dir): # Test successful delete check_image_dir.return_value = True self.image_manager.delete_image("ds1", "foo", 0, False) check_image_dir.assert_called_with("foo", "ds1") create_image_tombstone.assert_called_with("ds1", "foo") # Test successful delete with force option self.image_manager.delete_image("ds1", "foo", 0, True) check_image_dir.assert_called_with("foo", "ds1") create_image_tombstone.assert_called_with("ds1", "foo") lock_data_disk.assert_called_with("ds1", "foo") gc_image_dir.assert_called_with("ds1", "foo") clean_gc_dir.assert_called() # Test image not found check_image_dir.return_value = False self.assertRaises(ImageNotFoundException, self.image_manager.delete_image, "ds1", "foo", 0, False) @patch("host.hypervisor.esx.image_manager.os_vmdk_path") @patch("host.hypervisor.esx.image_manager.os_datastore_path") def test_gc_image_dir(self, dst_path, src_path): """ Test that we move the directory correctly to the GC location """ src_dir = file_util.mkdtemp(delete=True) dst_dir = file_util.mkdtemp(delete=True) src_path.return_value = os.path.join(src_dir, "test.vmdk") dst_path.return_value = dst_dir self.image_manager._gc_image_dir("ds1", "foo") uuid_dir = os.path.join(dst_dir, os.listdir(dst_dir)[0]) # Verify the src directory has been moved into the garbage dir. self.assertEqual(os.listdir(uuid_dir), [os.path.basename(src_dir)]) src_path.assert_called_once_with("ds1", "foo", IMAGE_FOLDER_NAME) dst_path.assert_called_once_with("ds1", GC_IMAGE_FOLDER) def test_image_path(self): image_path = "/vmfs/volumes/ds/images/tt/ttylinux/ttylinux.vmdk" ds = self.image_manager.get_datastore_id_from_path(image_path) image = self.image_manager.get_image_id_from_path(image_path) self.assertEqual(ds, "ds") self.assertEqual(image, "ttylinux") @patch("host.hypervisor.esx.image_manager.os_vmdk_flat_path") @patch("host.hypervisor.esx.image_manager.os.remove") def test_lock_data_disk(self, mock_rm, vmdk_flat_path): """ Test acquisition of the lock on the flat file. """ vmdk_flat_path.return_value = "fake_f_name" self.assertTrue(self.image_manager._lock_data_disk("ds1", "foo")) vmdk_flat_path.assert_called_once_with("ds1", "foo") mock_rm.side_effect = OSError self.assertFalse(self.image_manager._lock_data_disk("ds1", "foo")) @parameterized.expand([ ("CLOUD", "EAGER", ImageType.CLOUD, ImageReplication.EAGER), ("MANAGEMENT", "EAGER", ImageType.MANAGEMENT, ImageReplication.EAGER), ("CLOUD", "ON_DEMAND", ImageType.CLOUD, ImageReplication.ON_DEMAND), ("MANAGEMENT", "ON_DEMAND", ImageType.MANAGEMENT, ImageReplication.ON_DEMAND), ]) def test_image_type(self, type, replication, expected_type, expected_replication): self.ds_manager.image_datastores.return_value = ["ds1", "ds2"] with patch("host.hypervisor.esx.image_manager.os_image_manifest_path" "") as manifest_path: tmpdir = file_util.mkdtemp(delete=True) tmpfile = os.path.join(tmpdir, "ds1.manifest") manifest_path.return_value = tmpfile with open(tmpfile, 'w+') as f: f.write('{"imageType":"%s","imageReplication":"%s"}' % ( type, replication)) type, replication = self.image_manager.get_image_manifest( "image_id") self.assertEqual(type, expected_type) self.assertEqual(replication, expected_replication) def test_image_type_not_exist(self): self.ds_manager.image_datastores.return_value = ["ds1", "ds2"] type, replication = self.image_manager.get_image_manifest( "image_id") self.assertEqual(type, None) self.assertEqual(replication, None) @patch.object(EsxImageManager, "_move_image") @patch.object(EsxImageManager, "check_image_dir", return_value=False) @patch.object(EsxImageManager, "_create_image_timestamp_file_from_ids") @patch("os.path.exists") def test_create_image(self, _exists, _create_timestamp, check_image_dir, move_image): # Happy path verify move is called with the right args. _exists.side_effect = ([True]) self.image_manager.create_image("ds1", "foo", "img_1") check_image_dir.assert_called_once_with("img_1", "ds1") move_image.assert_called_once_with('img_1', 'ds1', '/vmfs/volumes/ds1/foo') _create_timestamp.assert_called_once_with("ds1", "img_1") # Verify error if tmp image doesn't exist _exists.side_effect = ([False]) move_image.reset_mock() self.assertRaises(ImageNotFoundException, self.image_manager.create_image, "ds1", "foo", "img_1") self.assertFalse(move_image.called) # Verify error if destination image already exists. _exists.side_effect = ([True]) move_image.reset_mock() check_image_dir.return_value = True self.assertRaises(DiskAlreadyExistException, self.image_manager.create_image, "ds1", "foo", "img_1") self.assertFalse(move_image.called) @patch.object(EsxImageManager, "create_image") @patch.object(EsxImageManager, "_manage_disk") @patch("os.path.exists", return_value=True) def test_create_image_with_vm_disk(self, _exists, _manage_disk, _create_image): vm_disk_path = "/vmfs/volumes/dsname/vms/ab/cd.vmdk" self.image_manager.create_image_with_vm_disk( "ds1", "foo", "img_1", vm_disk_path) # Verify that we copy the disk correctly expected_tmp_disk_ds_path = \ "[] /vmfs/volumes/ds1/foo/img_1.vmdk" _vd_spec = _manage_disk.call_args_list[0][1]['destSpec'] self.assertEqual("thin", _vd_spec.diskType) self.assertEqual("lsiLogic", _vd_spec.adapterType) copy_call = call(vim.VirtualDiskManager.CopyVirtualDisk_Task, sourceName='[] %s' % vm_disk_path, destName=expected_tmp_disk_ds_path, destSpec=_vd_spec) expected_vim_calls = [copy_call] self.assertEqual(expected_vim_calls, _manage_disk.call_args_list) _create_image.assert_called_once_with("ds1", "foo", "img_1") @patch("shutil.rmtree") @patch("os.path.exists") def test_delete_tmp_dir(self, _exists, _rmtree): self.image_manager.delete_tmp_dir("ds1", "foo") _exists.assert_called_once("/vmfs/volumes/ds1/foo") _rmtree.assert_called_once("/vmfs/volumes/ds1/foo") _exists.reset_mock() _exists.return_value = False _rmtree.reset_mock() self.assertRaises(DirectoryNotFound, self.image_manager.delete_tmp_dir, "ds1", "foo") _exists.assert_called_once("/vmfs/volumes/ds1/foo") self.assertFalse(_rmtree.called) def test_image_size(self): self.ds_manager.image_datastores.return_value = ["ds1", "ds2"] with patch("host.hypervisor.esx.image_manager.os_vmdk_flat_path" "") as image_path: tmpdir = file_util.mkdtemp(delete=True) image_path.return_value = tmpdir size = self.image_manager.image_size("image_id") self.assertTrue(size > 0) def test_image_size_not_exist(self): self.ds_manager.image_datastores.return_value = ["ds1", "ds2"] self.assertRaises(NoSuchResourceException, self.image_manager.image_size, "image_id")
class ImageSweeperTouchTimestampTestCase(unittest.TestCase): DATASTORE_ID = "DS01" BASE_TEMP_DIR = "image_sweeper" IMAGE_TIMESTAMP_FILENAME = EsxImageManager.IMAGE_TIMESTAMP_FILE_NAME IMAGE_TOMBSTONE_FILENAME = EsxImageManager.IMAGE_TOMBSTONE_FILE_NAME def setUp(self): self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.vm_manager = MagicMock() self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID) self.image_sweeper._task_runner = MagicMock() self.image_sweeper._task_runner.is_stopped.return_value = False self.delete_count = 0 # Create various image directories and empty vmdks dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "images/im") self.dir0 = dir0 # Image dir with correct timestamp file image_id_1 = str(uuid.uuid4()) dir1 = self.create_dir(image_id_1) open(os.path.join( dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close() # Image dir without the correct timestamp file image_id_2 = str(uuid.uuid4()) dir2 = self.create_dir(image_id_2) # Image dir with correct timestamp file # and with tombstone file image_id_3 = str(uuid.uuid4()) dir3 = self.create_dir(image_id_3) open(os.path.join( dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close() open(os.path.join( dir3, self.IMAGE_TOMBSTONE_FILENAME), 'w').close() self.image_ids = ["", image_id_1, image_id_2, image_id_3] self.image_dirs = ["", dir1, dir2, dir3] def tearDown(self): shutil.rmtree(self.test_dir, True) @parameterized.expand([ # timestamp_exists, tombstone_exists (True, False), (True, True), (False, False) ]) # The os_vmdk_path method is defined in vm_config.py # but it is imported in esx/image_manager.py, that is # the instance we need to patch @patch("host.hypervisor.esx.image_manager.os_vmdk_path") def test_touch_timestamp_file(self, timestamp_exists, tombstone_exists, os_vmdk_path): if tombstone_exists: image_index = 3 exception_class = type(InvalidImageState()) elif not timestamp_exists: image_index = 2 exception_class = type(OSError()) else: image_index = 1 exception_class = None image_id = self.image_ids[image_index] image_dir = self.image_dirs[image_index] os_vmdk_path.side_effect = self.patched_os_vmdk_path timestamp_filename_path = \ os.path.join(image_dir, self.IMAGE_TIMESTAMP_FILENAME) pre_mod_time = 0 if timestamp_exists: # save mod time on the image timestamp file pre_mod_time = os.path.getmtime(timestamp_filename_path) try: time.sleep(1) self.image_manager.\ touch_image_timestamp(self.DATASTORE_ID, image_id) assert_that(exception_class is None) # check new timestamp post_mod_time = os.path.getmtime(timestamp_filename_path) assert_that((post_mod_time > pre_mod_time) is True) except Exception as ex: assert_that((type(ex) == exception_class) is True) @patch("host.hypervisor.esx.image_manager.os_vmdk_path") def test_create_tombstone_file(self, os_vmdk_path): image_index = 1 image_id = self.image_ids[image_index] image_dir = self.image_dirs[image_index] os_vmdk_path.side_effect = self.patched_os_vmdk_path tombstone_filename_path = \ os.path.join(image_dir, self.IMAGE_TOMBSTONE_FILENAME) self.image_manager.create_image_tombstone(self.DATASTORE_ID, image_id) # check tombstone exists exists = os.path.exists(tombstone_filename_path) assert_that(exists is True) def patched_os_vmdk_path(self, datastore, disk_id, folder): folder = self.dir0 ret = os_vmdk_path(datastore, disk_id, folder) return ret def create_dir(self, image_id): dirname = os.path.join(self.dir0, _disk_path(image_id)) os.makedirs(dirname) return dirname
class EsxHypervisor(object): """Manage ESX Hypervisor.""" def __init__(self, agent_config): self.logger = logging.getLogger(__name__) # If VimClient's housekeeping thread failed to update its own cache, # call errback to commit suicide. Watchdog will bring up the agent # again. self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout, errback=lambda: suicide()) atexit.register(lambda client: client.disconnect(), self.vim_client) self._uuid = self.vim_client.host_uuid self.datastore_manager = EsxDatastoreManager( self, agent_config.datastores, agent_config.image_datastores) # datastore manager needs to update the cache when there is a change. self.vim_client.add_update_listener(self.datastore_manager) self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager) self.disk_manager = EsxDiskManager(self.vim_client, self.datastore_manager) self.image_manager = EsxImageManager(self.vim_client, self.datastore_manager) self.network_manager = EsxNetworkManager(self.vim_client, agent_config.networks) self.system = EsxSystem(self.vim_client) self.image_manager.monitor_for_cleanup() self.image_transferer = HttpNfcTransferer( self.vim_client, self.datastore_manager.image_datastores()) atexit.register(self.image_manager.cleanup) @property def uuid(self): return self._uuid def check_image(self, image_id, datastore_id): return self.image_manager.check_image( image_id, self.datastore_manager.datastore_name(datastore_id) ) def acquire_vim_ticket(self): return self.vim_client.acquire_clone_ticket() def add_update_listener(self, listener): self.vim_client.add_update_listener(listener) def remove_update_listener(self, listener): self.vim_client.remove_update_listener(listener) def transfer_image(self, source_image_id, source_datastore, destination_image_id, destination_datastore, host, port): return self.image_transferer.send_image_to_host( source_image_id, source_datastore, destination_image_id, destination_datastore, host, port) def prepare_receive_image(self, image_id, datastore): return self.image_manager.prepare_receive_image(image_id, datastore) def receive_image(self, image_id, datastore, imported_vm_name, metadata): self.image_manager.receive_image(image_id, datastore, imported_vm_name, metadata) def set_memory_overcommit(self, memory_overcommit): # Enable/Disable large page support. If this host is removed # from the deployment, large page support will need to be # explicitly updated by the user. disable_large_pages = memory_overcommit > 1.0 self.vim_client.set_large_page_support(disable=disable_large_pages)
class EsxHypervisor(object): """Manage ESX Hypervisor.""" def __init__(self, agent_config): self.logger = logging.getLogger(__name__) # If VimClient's housekeeping thread failed to update its own cache, # call errback to commit suicide. Watchdog will bring up the agent # again. errback = lambda: suicide() self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout, errback=errback) atexit.register(lambda client: client.disconnect(), self.vim_client) self._uuid = self.vim_client.host_uuid # Enable/Disable large page support. If this host is removed # from the deployment, large page support will need to be # explicitly updated by the user. disable_large_pages = agent_config.memory_overcommit > 1.0 self.vim_client.set_large_page_support(disable=disable_large_pages) image_datastores = [ds["name"] for ds in agent_config.image_datastores] self.datastore_manager = EsxDatastoreManager(self, agent_config.datastores, image_datastores) # datastore manager needs to update the cache when there is a change. self.vim_client.add_update_listener(self.datastore_manager) self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager) self.disk_manager = EsxDiskManager(self.vim_client, self.datastore_manager) self.image_manager = EsxImageManager(self.vim_client, self.datastore_manager) self.network_manager = EsxNetworkManager(self.vim_client, agent_config.networks) self.system = EsxSystem(self.vim_client) self.image_manager.monitor_for_cleanup() self.image_transferer = HttpNfcTransferer(self.vim_client, image_datastores) atexit.register(self.image_manager.cleanup) @property def uuid(self): return self._uuid @property def config(self): config = gen.hypervisor.esx.ttypes.EsxConfig() return TSerialization.serialize(config) def normalized_load(self): """ Return the maximum of the normalized memory/cpu loads""" memory = self.system.memory_info() memory_load = memory.used * 100 / memory.total # get average cpu load percentage in past 20 seconds # since hostd takes a sample in every 20 seconds # we use the min 20secs here to get the latest # CPU active average over 1 minute host_stats = copy.copy(self.vim_client.get_perf_manager_stats(20)) cpu_load = host_stats['rescpu.actav1'] / 100 return max(memory_load, cpu_load) def check_image(self, image_id, datastore_id): return self.image_manager.check_image( image_id, self.datastore_manager.datastore_name(datastore_id)) def acquire_vim_ticket(self): return self.vim_client.acquire_clone_ticket() def acquire_cgi_ticket(self, url, op): return self.vim_client.acquire_cgi_ticket(url, op) def add_update_listener(self, listener): self.vim_client.add_update_listener(listener) def remove_update_listener(self, listener): self.vim_client.remove_update_listener(listener) def transfer_image(self, source_image_id, source_datastore, destination_image_id, destination_datastore, host, port): return self.image_transferer.send_image_to_host( source_image_id, source_datastore, destination_image_id, destination_datastore, host, port) def receive_image(self, image_id, datastore, imported_vm_name): self.image_manager.receive_image(image_id, datastore, imported_vm_name)
class EsxHypervisor(object): """Manage ESX Hypervisor.""" def __init__(self, agent_config): self.logger = logging.getLogger(__name__) # If VimClient's housekeeping thread failed to update its own cache, # call errback to commit suicide. Watchdog will bring up the agent # again. errback = lambda: suicide() self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout, errback=errback) atexit.register(lambda client: client.disconnect(), self.vim_client) self._uuid = self.vim_client.host_uuid self.set_memory_overcommit(agent_config.memory_overcommit) image_datastores = [ds["name"] for ds in agent_config.image_datastores] self.datastore_manager = EsxDatastoreManager( self, agent_config.datastores, agent_config.image_datastores) # datastore manager needs to update the cache when there is a change. self.vim_client.add_update_listener(self.datastore_manager) self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager) self.disk_manager = EsxDiskManager(self.vim_client, self.datastore_manager) self.image_manager = EsxImageManager(self.vim_client, self.datastore_manager) self.network_manager = EsxNetworkManager(self.vim_client, agent_config.networks) self.system = EsxSystem(self.vim_client) self.image_manager.monitor_for_cleanup() self.image_transferer = HttpNfcTransferer(self.vim_client, image_datastores) atexit.register(self.image_manager.cleanup) @property def uuid(self): return self._uuid @property def config(self): config = gen.hypervisor.esx.ttypes.EsxConfig() return TSerialization.serialize(config) def normalized_load(self): """ Return the maximum of the normalized memory/cpu loads""" memory = self.system.memory_info() memory_load = memory.used * 100 / memory.total # get average cpu load percentage in past 20 seconds # since hostd takes a sample in every 20 seconds # we use the min 20secs here to get the latest # CPU active average over 1 minute host_stats = copy.copy(self.vim_client.get_perf_manager_stats(20)) cpu_load = host_stats['rescpu.actav1'] / 100 return max(memory_load, cpu_load) def check_image(self, image_id, datastore_id): return self.image_manager.check_image( image_id, self.datastore_manager.datastore_name(datastore_id) ) def acquire_vim_ticket(self): return self.vim_client.acquire_clone_ticket() def acquire_cgi_ticket(self, url, op): return self.vim_client.acquire_cgi_ticket(url, op) def add_update_listener(self, listener): self.vim_client.add_update_listener(listener) def remove_update_listener(self, listener): self.vim_client.remove_update_listener(listener) def transfer_image(self, source_image_id, source_datastore, destination_image_id, destination_datastore, host, port): return self.image_transferer.send_image_to_host( source_image_id, source_datastore, destination_image_id, destination_datastore, host, port) def receive_image(self, image_id, datastore, imported_vm_name): self.image_manager.receive_image( image_id, datastore, imported_vm_name) def set_memory_overcommit(self, memory_overcommit): # Enable/Disable large page support. If this host is removed # from the deployment, large page support will need to be # explicitly updated by the user. disable_large_pages = memory_overcommit > 1.0 self.vim_client.set_large_page_support(disable=disable_large_pages)