def setUp(self): self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.vm_manager = MagicMock() self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID) self.image_sweeper._task_runner = MagicMock() self.image_sweeper._task_runner.is_stopped.return_value = False self.delete_count = 0 # Create various image directories and empty vmdks dir0 = os.path.join(self.test_dir, self.DATASTORE_ID, "image_") self.dir0 = dir0 # Image dir with correct timestamp file image_id_1 = str(uuid.uuid4()) dir1 = self.create_dir(image_id_1) open(os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME), 'w').close() # Image dir without the correct timestamp file image_id_2 = str(uuid.uuid4()) dir2 = self.create_dir(image_id_2) # Image dir with correct timestamp file # and with tombstone file image_id_3 = str(uuid.uuid4()) dir3 = self.create_dir(image_id_3) open(os.path.join( dir3, self.IMAGE_TIMESTAMP_FILENAME), 'w').close() self.image_ids = ["", image_id_1, image_id_2, image_id_3] self.image_dirs = ["", dir1, dir2, dir3]
def __init__(self, agent_config): self.logger = logging.getLogger(__name__) # If VimClient's housekeeping thread failed to update its own cache, # call errback to commit suicide. Watchdog will bring up the agent # again. self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout, errback=lambda: suicide()) atexit.register(lambda client: client.disconnect(), self.vim_client) self._uuid = self.vim_client.host_uuid self.set_memory_overcommit(agent_config.memory_overcommit) self.datastore_manager = EsxDatastoreManager( self, agent_config.datastores, agent_config.image_datastores) # datastore manager needs to update the cache when there is a change. self.vim_client.add_update_listener(self.datastore_manager) self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager) self.disk_manager = EsxDiskManager(self.vim_client, self.datastore_manager) self.image_manager = EsxImageManager(self.vim_client, self.datastore_manager) self.network_manager = EsxNetworkManager(self.vim_client, agent_config.networks) self.system = EsxSystem(self.vim_client) self.image_manager.monitor_for_cleanup() self.image_transferer = HttpNfcTransferer( self.vim_client, self.datastore_manager.image_datastores()) atexit.register(self.image_manager.cleanup)
def test_reap_tmp_images(self, _os_datastore_path, _uuid): """ Test that stray images are found and deleted by the reaper """ def _fake_ds_folder(datastore, folder): return "%s__%s" % (datastore, folder) ds = MagicMock() ds.id = "dsid" ds.type = DatastoreType.EXT3 # In a random transient directory, set up a directory to act as the # tmp images folder and to contain a stray image folder with a file. tmpdir = file_util.mkdtemp(delete=True) tmp_images_folder = _fake_ds_folder(ds.id, TMP_IMAGE_FOLDER_NAME) tmp_images_dir = os.path.join(tmpdir, tmp_images_folder) tmp_image_dir = os.path.join(tmp_images_dir, "stray_image") os.mkdir(tmp_images_dir) os.mkdir(tmp_image_dir) (fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir) self.assertTrue(os.path.exists(path)) def _fake_os_datastore_path(datastore, folder): return os.path.join(tmpdir, _fake_ds_folder(datastore, folder)) _os_datastore_path.side_effect = _fake_os_datastore_path ds_manager = MagicMock() ds_manager.get_datastores.return_value = [ds] image_manager = EsxImageManager(self.vim_client, ds_manager) image_manager.reap_tmp_images() # verify stray image is deleted self.assertFalse(os.path.exists(path))
def setUp(self): self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.vm_manager = MagicMock() self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID) self.image_sweeper._task_runner = MagicMock() self.image_sweeper._task_runner.is_stopped.return_value = False self.delete_count = 0 # Create various image directories and empty vmdks dir0 = os.path.join(self.test_dir, "images/im") image_id_1 = str(uuid.uuid4()) image_id_2 = str(uuid.uuid4()) image_id_3 = str(uuid.uuid4()) image_id_4 = "invalid_image_id" self.image_ids = ["", image_id_1, image_id_2, image_id_3, image_id_4] dir1 = os.path.join(dir0, image_id_1) os.makedirs(dir1) dir2 = os.path.join(dir0, image_id_2) os.makedirs(dir2) dir3 = os.path.join(dir0, image_id_3) os.makedirs(dir3) dir4 = os.path.join(dir0, image_id_4) os.makedirs(dir4) # Create a vmdk under "im", since the image_id is # not a valid uuid it should be skipped open(os.path.join(dir0, "im.vmdk"), 'w').close() # Create a good image vmdk under image_id_1 but # no image marker file, this should not be deleted vmdk_filename = image_id_1 + ".vmdk" open(os.path.join(dir1, vmdk_filename), 'w').close() # Create a good image vmdk under image_id_2, also create # an unused image marker file, image_id_2 should be # deleted vmdk_filename = image_id_2 + ".vmdk" open(os.path.join(dir2, vmdk_filename), 'w').close() open(os.path.join(dir2, self.IMAGE_MARKER_FILENAME), 'w').close() # Create a marker file under dir3 but no # vmdk file. It should be deleted as well open(os.path.join(dir3, self.IMAGE_MARKER_FILENAME), 'w').close() # Create a vmdk under an invalid image directory, # also create a marker file. Since the image_id # is not valid it should not be deleted vmdk_filename = image_id_4 + ".vmdk" open(os.path.join(dir4, vmdk_filename), 'w').close() open(os.path.join(dir4, self.IMAGE_MARKER_FILENAME), 'w').close()
def setUp(self): self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.vm_manager = MagicMock() self.image_scanner = DatastoreImageScanner(self.image_manager, self.vm_manager, self.DATASTORE_ID) self.image_scanner._task_runner = MagicMock() self.image_scanner._task_runner.is_stopped.return_value = False self.write_count = 0 # Create various image directories and empty vmdks dir0 = os.path.join(self.test_dir, "images/im") image_id_1 = str(uuid.uuid4()) image_id_2 = str(uuid.uuid4()) image_id_3 = str(uuid.uuid4()) image_id_4 = "invalid_image_id" self.image_ids = ["", image_id_1, image_id_2, image_id_3, image_id_4] dir1 = os.path.join(dir0, image_id_1) os.makedirs(dir1) dir2 = os.path.join(dir0, image_id_2) os.makedirs(dir2) dir3 = os.path.join(dir0, image_id_3) os.makedirs(dir3) dir4 = os.path.join(dir0, image_id_4) os.makedirs(dir4) # Create a vmdk under "im", since the image_id is # not a valid uuid it should be skipped open(os.path.join(dir0, "im.vmdk"), 'w').close() # Create a good image vmdk under image_id_1, the name # of the vmdk matches the directory that contains it # so this is a valid image to remove vmdk_filename = image_id_1 + ".vmdk" open(os.path.join(dir1, vmdk_filename), 'w').close() # Create a good image vmdk under image_id_2, also create # an unused image marker file, image_id_2 should also be # included in the list of images to remove vmdk_filename = image_id_2 + ".vmdk" open(os.path.join(dir2, vmdk_filename), 'w').close() open(os.path.join(dir2, self.image_manager.IMAGE_MARKER_FILE_NAME), 'w').close() # Don't create anything under directory dir3 # it should still mark the image as deletable # Create a vmdk under an invalid image directory, since # the image id is not valid it should not mark it # for deletion vmdk_filename = image_id_4 + ".vmdk" open(os.path.join(dir4, vmdk_filename), 'w').close()
def test_periodic_reaper(self, mock_reap): """ Test that the we invoke the image reaper periodically """ image_manager = EsxImageManager(self.vim_client, self.ds_manager) image_manager.monitor_for_cleanup(reap_interval=0.1) self.assertFalse(image_manager._image_reaper is None) retry = 0 while mock_reap.call_count < 2 and retry < 10: time.sleep(0.1) retry += 1 image_manager.cleanup() assert_that(mock_reap.call_count, greater_than(1)) assert_that(retry, is_not(10), "reaper cleanup not called repeatedly")
def setUp(self, connect, update, creds): # Create VM manager creds.return_value = ["username", "password"] self.vim_client = VimClient(auto_sync=False) self.vim_client.wait_for_task = MagicMock() self.patcher = patch("host.hypervisor.esx.vm_config.GetEnv") self.patcher.start() self.vm_manager = EsxVmManager(self.vim_client, MagicMock()) services.register(ServiceName.AGENT_CONFIG, MagicMock()) # Set up test files self.base_dir = os.path.dirname(__file__) self.test_dir = os.path.join(self.base_dir, "../../test_files") self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.image_scanner = DatastoreImageScanner(self.image_manager, self.vm_manager, self.DATASTORE_ID) self.write_count = 0
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_path, _uuid): """ Test that stray images are found and deleted by the reaper """ def _fake_ds_folder(datastore, folder): return "%s/%s" % (datastore, folder) ds = MagicMock() ds.id = "dsid" ds.type = DatastoreType.EXT3 # In a random transient directory, set up a directory to act as the # tmp images folder and to contain a stray image folder with a file. tmpdir = file_util.mkdtemp(delete=True) tmp_ds_dir = os.path.join(tmpdir, ds.id) os.mkdir(tmp_ds_dir) tmp_image_dir = os.path.join( tmp_ds_dir, compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image")) os.mkdir(tmp_image_dir) (fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir) self.assertTrue(os.path.exists(path)) def _fake_os_datastore_path(datastore, folder): return os.path.join(tmpdir, _fake_ds_folder(datastore, folder)) _os_datastore_path.side_effect = _fake_os_datastore_path ds_manager = MagicMock() ds_manager.get_datastores.return_value = [ds] image_manager = EsxImageManager(self.vim_client, ds_manager) if not _allow_grace_period: image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0 time.sleep(0.1) image_manager.reap_tmp_images() if _allow_grace_period: # verify stray image is not deleted due to grace period self.assertTrue(os.path.exists(path)) else: # verify stray image is deleted self.assertFalse(os.path.exists(path))
def __init__(self, agent_config): self.logger = logging.getLogger(__name__) # If VimClient's housekeeping thread failed to update its own cache, # call errback to commit suicide. Watchdog will bring up the agent # again. errback = lambda: suicide() self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout, errback=errback) atexit.register(lambda client: client.disconnect(), self.vim_client) self._uuid = self.vim_client.host_uuid # Enable/Disable large page support. If this host is removed # from the deployment, large page support will need to be # explicitly updated by the user. disable_large_pages = agent_config.memory_overcommit > 1.0 self.vim_client.set_large_page_support(disable=disable_large_pages) image_datastores = [ds["name"] for ds in agent_config.image_datastores] self.datastore_manager = EsxDatastoreManager(self, agent_config.datastores, image_datastores) # datastore manager needs to update the cache when there is a change. self.vim_client.add_update_listener(self.datastore_manager) self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager) self.disk_manager = EsxDiskManager(self.vim_client, self.datastore_manager) self.image_manager = EsxImageManager(self.vim_client, self.datastore_manager) self.network_manager = EsxNetworkManager(self.vim_client, agent_config.networks) self.system = EsxSystem(self.vim_client) self.image_manager.monitor_for_cleanup() self.image_transferer = HttpNfcTransferer(self.vim_client, image_datastores) atexit.register(self.image_manager.cleanup)
def setUp(self): self.test_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) self.gc_dir = os.path.join(tempfile.mkdtemp(), self.BASE_TEMP_DIR) services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(MagicMock(), MagicMock()) self.vm_manager = MagicMock() self.image_sweeper = DatastoreImageSweeper(self.image_manager, self.DATASTORE_ID) self.deleted = False self.marker_unlinked = False # Create various image directories and empty vmdks image_id_1 = str(uuid.uuid4()) image_id_2 = str(uuid.uuid4()) image_id_3 = str(uuid.uuid4()) image_id_4 = str(uuid.uuid4()) self.image_id_1 = image_id_1 self.image_id_2 = image_id_2 self.image_id_3 = image_id_3 self.image_id_4 = image_id_4 dir1 = os.path.join(self.test_dir, "image_" + image_id_1) os.makedirs(dir1) dir2 = os.path.join(self.test_dir, "image_" + image_id_2) os.makedirs(dir2) dir3 = os.path.join(self.test_dir, "image_" + image_id_3) os.makedirs(dir3) dir4 = os.path.join(self.test_dir, "image_" + image_id_4) os.makedirs(dir4) self.marker_file_content_time = 0 self.timestamp_file_mod_time = 0 self.renamed_timestamp_file_mod_time = 0 # Create a good image vmdk under image_id_1, # also create a valid image marker file # and a valid timestamp file vmdk_filename = image_id_1 + ".vmdk" open(os.path.join(dir1, vmdk_filename), 'w').close() timestamp_filename = os.path.join(dir1, self.IMAGE_TIMESTAMP_FILENAME) open(timestamp_filename, 'w').close() marker_filename = os.path.join(dir1, self.IMAGE_MARKER_FILENAME) open(marker_filename, 'w').close() # Create a good image vmdk under image_id_2, # create timestamp but no image marker file, vmdk_filename = image_id_2 + ".vmdk" open(os.path.join(dir2, vmdk_filename), 'w').close() timestamp_filename = os.path.join(dir2, self.IMAGE_TIMESTAMP_FILENAME) open(timestamp_filename, 'w').close() # Create a good image vmdk under image_id_3, # create image_marker file but no timestamp # and no renamed timestamp file vmdk_filename = image_id_3 + ".vmdk" open(os.path.join(dir3, vmdk_filename), 'w').close() marker_filename = os.path.join(dir3, self.IMAGE_MARKER_FILENAME) open(marker_filename, 'w').close() # Create a good image vmdk under image_id_4, # create image_marker file, renamed timestamp file # but no timestamp file vmdk_filename = image_id_4 + ".vmdk" open(os.path.join(dir4, vmdk_filename), 'w').close() marker_filename = os.path.join(dir4, self.IMAGE_MARKER_FILENAME) open(marker_filename, 'w').close() timestamp_filename = os.path.join(dir4, self.IMAGE_TIMESTAMP_FILENAME) renamed_timestamp_filename = timestamp_filename + self.IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX open(renamed_timestamp_filename, 'w').close()
def setUp(self, connect, update, creds): creds.return_value = ["username", "password"] self.vim_client = VimClient(auto_sync=False) self.ds_manager = MagicMock() services.register(ServiceName.AGENT_CONFIG, MagicMock()) self.image_manager = EsxImageManager(self.vim_client, self.ds_manager)