def prepare_receive_image(self, image_id, datastore_id): ds_type = self._get_datastore_type(datastore_id) if ds_type == DatastoreType.VSAN: # on VSAN datastore, vm is imported to [vsanDatastore] image_[image_id]/[random_uuid].vmdk, # then the file is renamed to [vsanDatastore] image_[image_id]/[image_id].vmdk during receive_image. import_vm_path = datastore_path(datastore_id, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id)) self._vim_client.make_directory(import_vm_path) import_vm_id = str(uuid.uuid4()) else: # on other types of datastore, vm is imported to [datastore] tmp_image_[random_uuid]/[image_id].vmdk, # then the directory is renamed to [datastore] image_[image_id] during receive_image import_vm_path = datastore_path(datastore_id, compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4()))) import_vm_id = image_id return import_vm_path, import_vm_id
def delete_vm(self, vm_id, force=False): """Delete a Virtual Machine :param vm_id: Name of the VM :type vm_id: str :param force: Not to check persistent disk, forcefully delete vm. :type force: boolean :raise VmPowerStateException when vm is not powered off """ vm = self.vim_client.get_vm(vm_id) if vm.runtime.powerState != 'poweredOff': raise VmPowerStateException("Can only delete vm in state %s" % vm.runtime.powerState) # Getting the path for the new dir structure if we have upgraded from older structure datastore_name = self.get_vm_datastore(vm.config) vm_path = os_datastore_path(datastore_name, compond_path_join(VM_FOLDER_NAME_PREFIX, vm_id)) if not force: self._verify_disks(vm) self._logger.info("Destroy VM at %s" % vm_path) self._invoke_vm(vm, "Destroy") self._ensure_directory_cleanup(vm_path) self.vim_client.wait_for_vm_delete(vm_id)
def send_image_to_host(self, source_image_id, source_datastore, destination_image_id, destination_datastore, destination_host, destination_port): self._logger.info("transfer_image: connecting to remote agent") remote_agent_client = DirectClient("Host", Host.Client, destination_host, destination_port, 60) remote_agent_client.connect() self._logger.info("transfer_image: getting ticket") nfc_ticket = self._get_nfc_ticket(remote_agent_client, destination_datastore) self._logger.info("transfer_image: creating remote image") if destination_image_id is None: destination_image_id = source_image_id upload_folder = self._create_remote_image(remote_agent_client, destination_image_id, destination_datastore) try: source_file_path = datastore_path(source_datastore, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, source_image_id), vmdk_add_suffix(source_image_id)) destination_file_path = os.path.join(upload_folder, vmdk_add_suffix(destination_image_id)) self._logger.info("transfer_image: nfc copy image %s => (%s)%s, sslThumbprint=%s, ticket=%s", source_file_path, destination_host, destination_file_path, nfc_ticket.ssl_thumbprint, nfc_ticket.session_id) self._host_client.nfc_copy(source_file_path, destination_host, destination_file_path, nfc_ticket.ssl_thumbprint, nfc_ticket.session_id) self._logger.info("transfer_image: finalizing remote image") self._finalize_remote_image(remote_agent_client, destination_image_id, destination_datastore, upload_folder) except: self._logger.info("transfer_image: cleaning up failed transfer") self._cleanup_remote_image(remote_agent_client, destination_datastore, upload_folder) raise
def _copy_to_tmp_image(self, source_datastore, source_id, dest_datastore, dest_id): """ Copy an image into a temp location. 1. Lock a tmp image destination file with an exclusive lock. This is to prevent the GC thread from garbage collecting directories that are actively being used. The temp directory name contains a random UUID to prevent collisions with concurrent copies 2. Create the temp directory. 3. Copy the metadata file over. 4. Copy the vmdk over. @return the tmp image directory on success. """ ds_type = self._get_datastore_type(dest_datastore) if ds_type == DatastoreType.VSAN: tmp_image_dir = os_datastore_path(dest_datastore, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, dest_id), compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4()))) else: tmp_image_dir = os_datastore_path(dest_datastore, compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4()))) # Create the temp directory self._vim_client.make_directory(tmp_image_dir) # Copy the metadata file if it exists. source_meta = os_metadata_path(source_datastore, source_id, IMAGE_FOLDER_NAME_PREFIX) if os.path.exists(source_meta): try: dest_meta = os.path.join(tmp_image_dir, metadata_filename(dest_id)) shutil.copy(source_meta, dest_meta) except: self._logger.exception("Failed to copy metadata file %s", source_meta) raise # Create the timestamp file self._create_image_timestamp_file(tmp_image_dir) _vd_spec = self._prepare_virtual_disk_spec( vim.VirtualDiskManager.VirtualDiskType.thin, vim.VirtualDiskManager.VirtualDiskAdapterType.lsiLogic) self._manage_disk(vim.VirtualDiskManager.CopyVirtualDisk_Task, sourceName=vmdk_path(source_datastore, source_id, IMAGE_FOLDER_NAME_PREFIX), destName=os_to_datastore_path(os.path.join(tmp_image_dir, "%s.vmdk" % dest_id)), destSpec=_vd_spec) return tmp_image_dir
def create_image(self, image_id, datastore_id): """ Create a temp image on given datastore, return its path. """ datastore_type = self._get_datastore_type(datastore_id) if datastore_type == DatastoreType.VSAN: # on VSAN, tmp_dir is [datastore]/image_[image_id]/tmp_image_[uuid] # Because VSAN does not allow moving top-level directories, we place tmp_image # under image's dir. relative_path = os.path.join(compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id), compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4()))) tmp_dir = os_datastore_path(datastore_id, relative_path) else: # on VMFS/NFS/etc, tmp_dir is [datastore]/tmp_image_[uuid] tmp_dir = os_datastore_path(datastore_id, compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4()))) self._vim_client.make_directory(tmp_dir) # return datastore path, so that it can be passed to nfc client return os_to_datastore_path(tmp_dir)
def send_image_to_host(self, image_id, image_datastore, destination_image_id, destination_datastore, host, port): if destination_image_id is None: destination_image_id = image_id metadata = self._read_metadata(image_datastore, image_id) shadow_vm_id = self._create_shadow_vm() # place transfer.vmdk under shadow_vm_path to work around VSAN's restriction on # files at datastore top-level shadow_vm_path = os_datastore_path( self._get_shadow_vm_datastore(), compond_path_join(VM_FOLDER_NAME_PREFIX, shadow_vm_id)) transfer_vmdk_path = os.path.join(shadow_vm_path, "transfer.vmdk") self._logger.info("transfer_vmdk_path = %s" % transfer_vmdk_path) agent_client = None try: read_lease, disk_url = self._get_image_stream_from_shadow_vm( image_id, image_datastore, shadow_vm_id) try: self.download_file(disk_url, transfer_vmdk_path, read_lease) finally: read_lease.Complete() agent_client = DirectClient("Host", Host.Client, host, port) agent_client.connect() vm_path, vm_id = self._prepare_receive_image( agent_client, destination_image_id, destination_datastore) spec = self._create_import_vm_spec(vm_id, destination_datastore, vm_path) self._send_image(agent_client, host, transfer_vmdk_path, spec) self._register_imported_image_at_host(agent_client, destination_image_id, destination_datastore, vm_id, metadata) return vm_id finally: try: os.unlink(transfer_vmdk_path) except OSError: pass self._delete_shadow_vm(shadow_vm_id) rm_rf(shadow_vm_path) if agent_client: agent_client.close()
def send_image_to_host(self, image_id, image_datastore, destination_image_id, destination_datastore, host, port): if destination_image_id is None: destination_image_id = image_id metadata = self._read_metadata(image_datastore, image_id) shadow_vm_id = self._create_shadow_vm() tmp_path = "/vmfs/volumes/%s/%s_transfer.vmdk" % ( self._get_shadow_vm_datastore(), shadow_vm_id) self._logger.info("http_disk_transfer: tmp_path = %s" % tmp_path) agent_client = None try: read_lease, disk_url = self._get_image_stream_from_shadow_vm( image_id, image_datastore, shadow_vm_id) try: self.download_file(disk_url, tmp_path, read_lease) finally: read_lease.Complete() agent_client = DirectClient("Host", Host.Client, host, port) agent_client.connect() vm_path, vm_id = self._prepare_receive_image( agent_client, destination_image_id, destination_datastore) spec = self._create_import_vm_spec(vm_id, destination_datastore, vm_path) self._send_image(agent_client, host, tmp_path, spec) self._register_imported_image_at_host(agent_client, destination_image_id, destination_datastore, vm_id, metadata) return vm_id finally: try: os.unlink(tmp_path) except OSError: pass self._delete_shadow_vm(shadow_vm_id) rm_rf( os_datastore_path( self._get_shadow_vm_datastore(), compond_path_join(VM_FOLDER_NAME_PREFIX, shadow_vm_id))) if agent_client: agent_client.close()
def _move_image(self, image_id, datastore, tmp_dir): """ Atomic move of a tmp folder into the image datastore. Handles concurrent moves by locking a well know derivative of the image_id while doing the atomic move. The exclusive file lock ensures that only one move is successful. Has the following side effects: a - If the destination image already exists, it is assumed that someone else successfully copied the image over and the temp directory is deleted. b - If we fail to acquire the file lock after retrying 3 times, or the atomic move fails, the tmp image directory will be left behind and needs to be garbage collected later. image_id: String.The image id of the image being moved. datastore: String. The datastore id of the datastore. tmp_dir: String. The absolute path of the temp image directory. raises: OsError if the move fails AcquireLockFailure, InvalidFile if we fail to lock the destination image. """ ds_type = self._get_datastore_type(datastore) image_path = os_datastore_path(datastore, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id)) self._logger.info("_move_image: %s => %s, ds_type: %s" % (tmp_dir, image_path, ds_type)) if not os.path.exists(tmp_dir): raise ImageNotFoundException("Temp image %s not found" % tmp_dir) try: with FileBackedLock(image_path, ds_type, retry=300, wait_secs=0.01): # wait lock for 3 seconds if self._check_image_repair(image_id, datastore): raise DiskAlreadyExistException("Image already exists") if ds_type == DatastoreType.VSAN: # on VSAN, move all files under [datastore]/image_[image_id]/tmp_image_[uuid]/* to # [datastore]/image_[image_id]/*. # Also we do not delete tmp_image folder in success case, because VSAN accesses it # when creating linked VM, even the folder is now empty. for entry in os.listdir(tmp_dir): shutil.move(os.path.join(tmp_dir, entry), os.path.join(image_path, entry)) else: # on VMFS/NFS/etc, rename [datastore]/tmp_image_[uuid] to [datastore]/tmp_image_[image_id] self._vim_client.move_file(tmp_dir, image_path) except: self._logger.exception("Move image %s to %s failed" % (image_id, image_path)) self._vim_client.delete_file(tmp_dir) raise
def send_image_to_host(self, source_image_id, source_datastore, destination_image_id, destination_datastore, destination_host, destination_port): self._logger.info("transfer_image: connecting to remote agent") remote_agent_client = DirectClient("Host", Host.Client, destination_host, destination_port, 60) remote_agent_client.connect() self._logger.info("transfer_image: getting ticket") nfc_ticket = self._get_nfc_ticket(remote_agent_client, destination_datastore) self._logger.info("transfer_image: creating remote image") if destination_image_id is None: destination_image_id = source_image_id upload_folder = self._create_remote_image(remote_agent_client, destination_image_id, destination_datastore) try: source_file_path = datastore_path( source_datastore, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, source_image_id), vmdk_add_suffix(source_image_id)) destination_file_path = os.path.join( upload_folder, vmdk_add_suffix(destination_image_id)) self._logger.info( "transfer_image: nfc copy image %s => (%s)%s, sslThumbprint=%s, ticket=%s", source_file_path, destination_host, destination_file_path, nfc_ticket.ssl_thumbprint, nfc_ticket.session_id) self._host_client.nfc_copy(source_file_path, destination_host, destination_file_path, nfc_ticket.ssl_thumbprint, nfc_ticket.session_id) self._logger.info("transfer_image: finalizing remote image") self._finalize_remote_image(remote_agent_client, destination_image_id, destination_datastore, upload_folder) except: self._logger.info("transfer_image: cleaning up failed transfer") self._cleanup_remote_image(remote_agent_client, destination_datastore, upload_folder) raise
def _delete_unused_images(self, image_sweeper, datastore_root): deleted_images = list() target_images = image_sweeper.get_target_images() # Compute sweep rest interval rest_interval_sec = image_sweeper.get_image_sweep_rest_interval() for image_id in target_images: # On a directory change check if it still needs to run if image_sweeper.is_stopped(): return image_dir = os.path.join( datastore_root, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id)) # If there is not a marker file, skip it marker_pathname = os.path.join( image_dir, self._image_manager.IMAGE_MARKER_FILE_NAME) if not os.path.isfile(marker_pathname): self._logger.warn( "skipping image(%s) because marker file not found" % image_id) continue try: if self._image_manager._delete_single_image( image_sweeper, image_dir, image_id): deleted_images.append(image_id) except Exception as ex: self._logger.warning("Failed to remove image: %s, %s" % (image_dir, ex)) continue waste_time(rest_interval_sec) # Now attempt GCing the image directory. try: self._image_manager._clean_gc_dir(image_sweeper.datastore_id) except Exception: # Swallow the exception the next clean call will clear it all. self._logger.exception("Failed to delete gc dir on datastore %s" % image_sweeper.datastore_id) return deleted_images
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_path, _uuid): """ Test that stray images are found and deleted by the reaper """ def _fake_ds_folder(datastore, folder): return "%s/%s" % (datastore, folder) ds = MagicMock() ds.id = "dsid" ds.type = DatastoreType.EXT3 # In a random transient directory, set up a directory to act as the # tmp images folder and to contain a stray image folder with a file. tmpdir = file_util.mkdtemp(delete=True) tmp_ds_dir = os.path.join(tmpdir, ds.id) os.mkdir(tmp_ds_dir) tmp_image_dir = os.path.join( tmp_ds_dir, compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image")) os.mkdir(tmp_image_dir) (fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir) self.assertTrue(os.path.exists(path)) def _fake_os_datastore_path(datastore, folder): return os.path.join(tmpdir, _fake_ds_folder(datastore, folder)) _os_datastore_path.side_effect = _fake_os_datastore_path ds_manager = MagicMock() ds_manager.get_datastores.return_value = [ds] image_manager = EsxImageManager(self.vim_client, ds_manager) if not _allow_grace_period: image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0 time.sleep(0.1) image_manager.reap_tmp_images() if _allow_grace_period: # verify stray image is not deleted due to grace period self.assertTrue(os.path.exists(path)) else: # verify stray image is deleted self.assertFalse(os.path.exists(path))
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_path, _uuid): """ Test that stray images are found and deleted by the reaper """ def _fake_ds_folder(datastore, folder): return "%s/%s" % (datastore, folder) ds = MagicMock() ds.id = "dsid" ds.type = DatastoreType.EXT3 # In a random transient directory, set up a directory to act as the # tmp images folder and to contain a stray image folder with a file. tmpdir = file_util.mkdtemp(delete=True) tmp_ds_dir = os.path.join(tmpdir, ds.id) os.mkdir(tmp_ds_dir) tmp_image_dir = os.path.join(tmp_ds_dir, compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image")) os.mkdir(tmp_image_dir) (fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir) self.assertTrue(os.path.exists(path)) def _fake_os_datastore_path(datastore, folder): return os.path.join(tmpdir, _fake_ds_folder(datastore, folder)) _os_datastore_path.side_effect = _fake_os_datastore_path ds_manager = MagicMock() ds_manager.get_datastores.return_value = [ds] image_manager = EsxImageManager(self.vim_client, ds_manager) if not _allow_grace_period: image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0 time.sleep(0.1) image_manager.reap_tmp_images() if _allow_grace_period: # verify stray image is not deleted due to grace period self.assertTrue(os.path.exists(path)) else: # verify stray image is deleted self.assertFalse(os.path.exists(path))
def send_image_to_host(self, image_id, image_datastore, destination_image_id, destination_datastore, host, port): manifest, metadata = self._read_metadata(image_datastore, image_id) shadow_vm_id = self._create_shadow_vm() tmp_path = "/vmfs/volumes/%s/%s_transfer.vmdk" % ( self._get_shadow_vm_datastore(), shadow_vm_id) try: read_lease, disk_url =\ self._get_image_stream_from_shadow_vm( image_id, image_datastore, shadow_vm_id) try: self.download_file(disk_url, tmp_path, read_lease) finally: read_lease.Complete() if destination_image_id is None: destination_image_id = image_id spec = self._create_import_vm_spec( destination_image_id, destination_datastore) imported_vm_name = self._send_image( tmp_path, manifest, metadata, spec, destination_image_id, destination_datastore, host, port) return imported_vm_name finally: try: os.unlink(tmp_path) except OSError: pass self._delete_shadow_vm(shadow_vm_id) rm_rf(os_datastore_path(self._get_shadow_vm_datastore(), compond_path_join(VM_FOLDER_NAME_PREFIX, shadow_vm_id)))
def _delete_unused_images(self, image_sweeper, datastore_root): deleted_images = list() target_images = image_sweeper.get_target_images() # Compute sweep rest interval rest_interval_sec = image_sweeper.get_image_sweep_rest_interval() for image_id in target_images: # On a directory change check if it still needs to run if image_sweeper.is_stopped(): return image_dir = os.path.join(datastore_root, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id)) # If there is not a marker file, skip it marker_pathname = os.path.join(image_dir, self._image_manager.IMAGE_MARKER_FILE_NAME) if not os.path.isfile(marker_pathname): self._logger.warn("skipping image(%s) because marker file not found" % image_id) continue try: if self._image_manager._delete_single_image(image_sweeper, image_dir, image_id): deleted_images.append(image_id) except Exception as ex: self._logger.warning("Failed to remove image: %s, %s" % (image_dir, ex)) continue waste_time(rest_interval_sec) # Now attempt GCing the image directory. try: self._image_manager._clean_gc_dir(image_sweeper.datastore_id) except Exception: # Swallow the exception the next clean call will clear it all. self._logger.exception("Failed to delete gc dir on datastore %s" % image_sweeper.datastore_id) return deleted_images
def create_dir(self, image_id): dirname = vm_config.compond_path_join(self.dir0, image_id) os.makedirs(dirname) return dirname
def _delete_image(self, image, result_code=DeleteDirectoryResultCode.OK): resp = self.host_client.delete_directory( DeleteDirectoryRequest(image.datastore.id, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image.id))) assert_that(resp.result, is_(result_code))