def test_get_linked_clone_image_path(self):
image_path = self.vm_manager.get_linked_clone_image_path
# VM not found
vm = MagicMock(return_value=None)
self.vm_manager.vim_client.get_vm_in_cache = vm
assert_that(image_path("vm1"), is_(None))
# disks is None
vm = MagicMock(return_value=VmCache(disks=None))
self.vm_manager.vim_client.get_vm_in_cache = vm
assert_that(image_path("vm1"), is_(None))
# disks is an empty list
vm = MagicMock(return_value=VmCache(disks=[]))
self.vm_manager.vim_client.get_vm_in_cache = vm
assert_that(image_path("vm1"), is_(None))
# no image disk
vm = MagicMock(return_value=VmCache(disks=["a", "b", "c"]))
self.vm_manager.vim_client.get_vm_in_cache = vm
assert_that(image_path("vm1"), is_(None))
# image found
image = "[ds1] image_ttylinux/ttylinux.vmdk"
vm = MagicMock(return_value=VmCache(disks=["a", "b", image]))
self.vm_manager.vim_client.get_vm_in_cache = vm
assert_that(image_path("vm1"), is_(datastore_to_os_path(image)))
def create_vm(self, vm_id, create_spec):
"""Create a new Virtual Maching given a VM create spec.
:param vm_id: The Vm id
:type vm_id: string
:param create_spec: The VM spec builder
:type ConfigSpec
:raise: VmAlreadyExistException
"""
folder = self.vim_client.vm_folder
resource_pool = self.vim_client.root_resource_pool
# sanity check since VIM does not prevent this
try:
if self.vim_client.get_vm_in_cache(vm_id):
raise VmAlreadyExistException("VM already exists")
except VmNotFoundException:
pass
# The scenario of the vm creation at ESX where intermediate directory
# has to be created has not been well exercised and is known to be
# racy and not informative on failures. So be defensive and proactively
# create the intermediate directory ("/vmfs/volumes/<dsid>/vms/xy").
vm_parent_dir = datastore_to_os_path(create_spec.files.vmPathName)
if os.path.exists(vm_parent_dir):
self._logger.debug("Parent directory %s exists" % vm_parent_dir)
else:
mkdir_p(vm_parent_dir)
task = folder.CreateVm(create_spec, resource_pool, None)
self.vim_client.wait_for_task(task)
self.vim_client.wait_for_vm_create(vm_id)
def create_vm(self, vm_id, create_spec):
"""Create a new Virtual Maching given a VM create spec.
:param vm_id: The Vm id
:type vm_id: string
:param create_spec: The VM spec builder
:type ConfigSpec
:raise: VmAlreadyExistException
"""
folder = self.vim_client.vm_folder
resource_pool = self.vim_client.root_resource_pool
# sanity check since VIM does not prevent this
try:
if self.vim_client.get_vm_in_cache(vm_id):
raise VmAlreadyExistException("VM already exists")
except VmNotFoundException:
pass
# The scenario of the vm creation at ESX where intermediate directory
# has to be created has not been well exercised and is known to be
# racy and not informative on failures. So be defensive and proactively
# create the intermediate directory ("/vmfs/volumes/<dsid>/vms/xy").
vm_parent_dir = datastore_to_os_path(create_spec.files.vmPathName)
if os.path.exists(vm_parent_dir):
self._logger.debug("Parent directory %s exists" % vm_parent_dir)
else:
mkdir_p(vm_parent_dir)
task = folder.CreateVm(create_spec, resource_pool, None)
self.vim_client.wait_for_task(task)
self.vim_client.wait_for_vm_create(vm_id)
def test_get_linked_clone_image_path(self):
image_path = self.vm_manager.get_linked_clone_image_path
# VM not found
vm = MagicMock(return_value=None)
self.vm_manager.vim_client.get_vm_in_cache = vm
assert_that(image_path("vm1"), is_(None))
# disks is None
vm = MagicMock(return_value=VmCache(disks=None))
self.vm_manager.vim_client.get_vm_in_cache = vm
assert_that(image_path("vm1"), is_(None))
# disks is an empty list
vm = MagicMock(return_value=VmCache(disks=[]))
self.vm_manager.vim_client.get_vm_in_cache = vm
assert_that(image_path("vm1"), is_(None))
# no image disk
vm = MagicMock(return_value=VmCache(disks=["a", "b", "c"]))
self.vm_manager.vim_client.get_vm_in_cache = vm
assert_that(image_path("vm1"), is_(None))
# image found
image = "[ds1] image_ttylinux/ttylinux.vmdk"
vm = MagicMock(return_value=VmCache(disks=["a", "b", image]))
self.vm_manager.vim_client.get_vm_in_cache = vm
assert_that(image_path("vm1"), is_(datastore_to_os_path(image)))
def receive_image(self, image_id, datastore_id, imported_vm_name, metadata, manifest):
""" Creates an image using the data from the imported vm.
This is run at the destination host end of the host-to-host
image transfer.
"""
vm = self._vim_client.get_vm_obj_in_cache(imported_vm_name)
vmx_os_path = datastore_to_os_path(vm.config.files.vmPathName)
vm_dir = os.path.dirname(vmx_os_path)
vm.Unregister()
if self.check_image_dir(image_id, datastore_id):
self._logger.info("Image %s on datastore %s already exists" % (image_id, datastore_id))
raise DiskAlreadyExistException()
self._move_image(image_id, datastore_id, vm_dir)
# Save raw manifest
manifest_path = os_image_manifest_path(datastore_id, image_id)
with open(manifest_path, "w") as f:
f.write(manifest)
# Save raw metadata
metadata_path = os_metadata_path(datastore_id, image_id, IMAGE_FOLDER_NAME)
with open(metadata_path, "w") as f:
f.write(metadata)
self._create_image_timestamp_file_from_ids(datastore_id, image_id)
def create_image_with_vm_disk(self, datastore_id, tmp_dir, image_id,
vm_disk_os_path):
""" Fills a temp image directory with a disk from a VM,
then installs directory in the shared image folder.
"""
# Create parent directory as required by CopyVirtualDisk_Task
dst_vmdk_path = os.path.join(datastore_to_os_path(tmp_dir), "%s.vmdk" % image_id)
if os.path.exists(dst_vmdk_path):
self._logger.warning(
"Unexpected disk %s present, overwriting" % dst_vmdk_path)
dst_vmdk_ds_path = os_to_datastore_path(dst_vmdk_path)
_vd_spec = self._prepare_virtual_disk_spec(
vim.VirtualDiskManager.VirtualDiskType.thin,
vim.VirtualDiskManager.VirtualDiskAdapterType.lsiLogic)
self._manage_disk(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName=os_to_datastore_path(vm_disk_os_path),
destName=dst_vmdk_ds_path,
destSpec=_vd_spec)
try:
self.finalize_image(datastore_id, tmp_dir, image_id)
except:
self._logger.warning("Delete copied disk %s" % dst_vmdk_ds_path)
self._manage_disk(vim.VirtualDiskManager.DeleteVirtualDisk_Task,
name=dst_vmdk_ds_path)
raise
def create_image_with_vm_disk(self, datastore_id, tmp_dir, image_id,
vm_disk_os_path):
""" Fills a temp image directory with a disk from a VM,
then installs directory in the shared image folder.
"""
# Create parent directory as required by CopyVirtualDisk_Task
dst_vmdk_path = os.path.join(datastore_to_os_path(tmp_dir), "%s.vmdk" % image_id)
if os.path.exists(dst_vmdk_path):
self._logger.warning(
"Unexpected disk %s present, overwriting" % dst_vmdk_path)
dst_vmdk_ds_path = os_to_datastore_path(dst_vmdk_path)
_vd_spec = self._prepare_virtual_disk_spec(
vim.VirtualDiskManager.VirtualDiskType.thin,
vim.VirtualDiskManager.VirtualDiskAdapterType.lsiLogic)
self._manage_disk(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName=os_to_datastore_path(vm_disk_os_path),
destName=dst_vmdk_ds_path,
destSpec=_vd_spec)
try:
self.finalize_image(datastore_id, tmp_dir, image_id)
except:
self._logger.warning("Delete copied disk %s" % dst_vmdk_ds_path)
self._manage_disk(vim.VirtualDiskManager.DeleteVirtualDisk_Task,
name=dst_vmdk_ds_path)
raise
def _create_tmp_image(self, source_datastore, source_id, dest_datastore, dest_id):
""" Copy an image into a temp location.
1. Lock a tmp image destination file with an exclusive lock. This
is to prevent the GC thread from garbage collecting directories
that are actively being used.
The temp directory name contains a random UUID to prevent
collisions with concurrent copies
2. Create the temp directory.
3. Copy the metadata file over.
4. Copy the vmdk over.
@return the tmp image directory on success.
"""
source = vmdk_path(source_datastore, source_id, IMAGE_FOLDER_NAME)
temp_dest = tmp_image_path(dest_datastore, dest_id)
ds_type = self._get_datastore_type(dest_datastore)
tmp_image_dir_path = os.path.dirname(datastore_to_os_path(temp_dest))
# Try grabbing the lock on the temp directory if it fails
# (very unlikely) someone else is copying an image just retry
# later.
with FileBackedLock(tmp_image_dir_path, ds_type):
source_meta = os_metadata_path(source_datastore, source_id, IMAGE_FOLDER_NAME)
# Create the temp directory
mkdir_p(tmp_image_dir_path)
# Copy the metadata file if it exists.
if os.path.exists(source_meta):
try:
dest_meta = os.path.join(tmp_image_dir_path, metadata_filename(dest_id))
shutil.copy(source_meta, dest_meta)
except:
self._logger.exception("Failed to copy metadata file %s", source_meta)
raise
# Copy the manifest file if it exists
source_manifest = os_image_manifest_path(source_datastore, source_id)
if os.path.exists(source_manifest):
try:
dest_manifest = os.path.join(tmp_image_dir_path, manifest_filename(dest_id))
shutil.copy(source_manifest, dest_manifest)
except:
# Swallow it. Not critical.
pass
# Create the timestamp file
self._create_image_timestamp_file(tmp_image_dir_path)
_vd_spec = self._prepare_virtual_disk_spec(
vim.VirtualDiskManager.VirtualDiskType.thin, vim.VirtualDiskManager.VirtualDiskAdapterType.lsiLogic
)
self._manage_disk(
vim.VirtualDiskManager.CopyVirtualDisk_Task, sourceName=source, destName=temp_dest, destSpec=_vd_spec
)
return tmp_image_dir_path
def _ensure_directory_cleanup(self, vm_ds_path):
# Upon successful destroy of VM, log any stray files still left in the
# VM directory and delete the directory.
vm_dir = os.path.dirname(datastore_to_os_path(vm_ds_path))
if os.path.isdir(vm_dir):
files = os.listdir(vm_dir)
for f in files:
if f.endswith(".vmdk"):
self._logger.info("Stray disk "
"(possible data leak): %s" % f)
else:
self._logger.info("Stray file: %s" % f)
self._logger.warning("Force delete vm directory %s" % vm_dir)
rm_rf(vm_dir)
def _ensure_directory_cleanup(self, vm_ds_path):
# Upon successful destroy of VM, log any stray files still left in the
# VM directory and delete the directory.
vm_dir = os.path.dirname(datastore_to_os_path(vm_ds_path))
if os.path.isdir(vm_dir):
files = os.listdir(vm_dir)
for f in files:
if f.endswith(".vmdk"):
self._logger.info("Stray disk "
"(possible data leak): %s" % f)
else:
self._logger.info("Stray file: %s" % f)
self._logger.warning("Force delete vm directory %s" % vm_dir)
rm_rf(vm_dir)
def _create_tmp_image(self, source_datastore, source_id, dest_datastore,
dest_id):
""" Copy an image into a temp location.
1. Lock a tmp image destination file with an exclusive lock. This
is to prevent the GC thread from garbage collecting directories
that are actively being used.
The temp directory name contains a random UUID to prevent
collisions with concurrent copies
2. Create the temp directory.
3. Copy the metadata file over.
4. Copy the vmdk over.
@return the tmp image directory on success.
"""
source = vmdk_path(source_datastore, source_id, IMAGE_FOLDER_NAME)
temp_dest = tmp_image_path(dest_datastore, dest_id)
ds_type = self._get_datastore_type(dest_datastore)
tmp_image_dir_path = os.path.dirname(datastore_to_os_path(temp_dest))
# Try grabbing the lock on the temp directory if it fails
# (very unlikely) someone else is copying an image just retry
# later.
with FileBackedLock(tmp_image_dir_path, ds_type):
source_meta = os_metadata_path(source_datastore, source_id,
IMAGE_FOLDER_NAME)
# Create the temp directory
mkdir_p(tmp_image_dir_path)
# Copy the metadata file if it exists.
if os.path.exists(source_meta):
try:
shutil.copy(source_meta, tmp_image_dir_path)
except:
self._logger.exception("Failed to copy metadata file %s",
source_meta)
raise
# Create the timestamp file
self._create_image_timestamp_file(tmp_image_dir_path)
_vd_spec = self._prepare_virtual_disk_spec(
vim.VirtualDiskManager.VirtualDiskType.thin,
vim.VirtualDiskManager.VirtualDiskAdapterType.lsiLogic)
self._manage_disk(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName=source,
destName=temp_dest,
destSpec=_vd_spec)
return tmp_image_dir_path
def receive_image(self, image_id, datastore_id, imported_vm_name):
""" Creates an image using the data from the imported vm.
This is run at the destination host end of the host-to-host
image transfer.
"""
vm = self._vim_client.get_vm_obj_in_cache(imported_vm_name)
vmx_os_path = datastore_to_os_path(vm.config.files.vmPathName)
vm_dir = os.path.dirname(vmx_os_path)
vm.Unregister()
if self.check_image_dir(image_id, datastore_id):
self._logger.info("Image %s on datastore %s already exists" %
(image_id, datastore_id))
raise DiskAlreadyExistException()
self._move_image(image_id, datastore_id, vm_dir)
self._create_image_timestamp_file_from_ids(datastore_id, image_id)
def receive_image(self, image_id, datastore_id, imported_vm_name):
""" Creates an image using the data from the imported vm.
This is run at the destination host end of the host-to-host
image transfer.
"""
vm = self._vim_client.get_vm_obj_in_cache(imported_vm_name)
vmx_os_path = datastore_to_os_path(vm.config.files.vmPathName)
vm_dir = os.path.dirname(vmx_os_path)
vm.Unregister()
if self.check_image_dir(image_id, datastore_id):
self._logger.info("Image %s on datastore %s already exists" %
(image_id, datastore_id))
raise DiskAlreadyExistException()
self._move_image(image_id, datastore_id, vm_dir)
self._create_image_timestamp_file_from_ids(datastore_id, image_id)
def receive_image(self, image_id, datastore_id, imported_vm_name, metadata):
""" Creates an image using the data from the imported vm.
This is run at the destination host end of the host-to-host
image transfer.
"""
self._vim_client.wait_for_vm_create(imported_vm_name)
vm = self._vim_client.get_vm_obj_in_cache(imported_vm_name)
self._logger.warning("receive_image found vm %s, %s" % (imported_vm_name, vm))
vm_dir = os.path.dirname(datastore_to_os_path(vm.config.files.vmPathName))
vm.Unregister()
ds_type = self._get_datastore_type(datastore_id)
if ds_type == DatastoreType.VSAN:
# on VSAN datastore, vm_dir is [vsanDatastore] image_[image_id], we only need to
# rename the vmdk file to [image_id].vmdk
try:
with FileBackedLock(vm_dir, ds_type, retry=300, wait_secs=0.01): # wait lock for 3 seconds
if self._check_image_repair(image_id, datastore_id):
raise DiskAlreadyExistException("Image already exists")
self._vim_client.move_file(os.path.join(vm_dir, vmdk_add_suffix(imported_vm_name)),
os.path.join(vm_dir, vmdk_add_suffix(image_id)))
except:
self._logger.exception("Move image %s to %s failed" % (image_id, vm_dir))
self._vim_client.delete_file(vm_dir)
raise
else:
self._move_image(image_id, datastore_id, vm_dir)
# Save raw metadata
if metadata:
metadata_path = os_metadata_path(datastore_id, image_id, IMAGE_FOLDER_NAME_PREFIX)
with open(metadata_path, 'w') as f:
f.write(metadata)
self._create_image_timestamp_file_from_ids(datastore_id, image_id)
def receive_image(self, image_id, datastore_id, imported_vm_name, metadata):
""" Creates an image using the data from the imported vm.
This is run at the destination host end of the host-to-host
image transfer.
"""
self._vim_client.wait_for_vm_create(imported_vm_name)
vm = self._vim_client.get_vm_obj_in_cache(imported_vm_name)
self._logger.warning("receive_image found vm %s, %s" % (imported_vm_name, vm))
vm_dir = os.path.dirname(datastore_to_os_path(vm.config.files.vmPathName))
vm.Unregister()
ds_type = self._get_datastore_type(datastore_id)
if ds_type == DatastoreType.VSAN:
# on VSAN datastore, vm_dir is [vsanDatastore] image_[image_id], we only need to
# rename the vmdk file to [image_id].vmdk
try:
with FileBackedLock(vm_dir, ds_type, retry=300, wait_secs=0.01): # wait lock for 3 seconds
if self._check_image_repair(image_id, datastore_id):
raise DiskAlreadyExistException("Image already exists")
self._vim_client.move_file(os.path.join(vm_dir, vmdk_add_suffix(imported_vm_name)),
os.path.join(vm_dir, vmdk_add_suffix(image_id)))
except:
self._logger.exception("Move image %s to %s failed" % (image_id, vm_dir))
self._vim_client.delete_file(vm_dir)
raise
else:
self._move_image(image_id, datastore_id, vm_dir)
# Save raw metadata
if metadata:
metadata_path = os_metadata_path(datastore_id, image_id, IMAGE_FOLDER_NAME_PREFIX)
with open(metadata_path, 'w') as f:
f.write(metadata)
self._create_image_timestamp_file_from_ids(datastore_id, image_id)
def test_path_conversion(self, ds_path, expected_os_path):
path = datastore_to_os_path(ds_path)
assert_that(path, equal_to(expected_os_path))
def _vmdk_pairs(ds_path):
vmdk_path = datastore_to_os_path(ds_path)
pos = vmdk_path.rfind(".vmdk")
vmdk_flat_path = vmdk_path[:pos] + "-flat" + vmdk_path[pos:]
return (vmdk_path, vmdk_flat_path)
def _vmdk_pairs(ds_path):
vmdk_path = datastore_to_os_path(ds_path)
pos = vmdk_path.rfind(".vmdk")
vmdk_flat_path = vmdk_path[:pos] + "-flat" + vmdk_path[pos:]
return (vmdk_path, vmdk_flat_path)
def remove_iso(self, iso_ds_path):
try:
os.remove(datastore_to_os_path(iso_ds_path))
except:
# The iso may not exist, so just catch and move on.
pass
def remove_iso(self, iso_ds_path):
try:
os.remove(datastore_to_os_path(iso_ds_path))
except:
# The iso may not exist, so just catch and move on.
pass
def finalize_image(self, datastore_id, tmp_dir, image_id):
""" Installs an image using image data staged at a temp directory.
"""
self._move_image(image_id, datastore_id, datastore_to_os_path(tmp_dir))
self._create_image_timestamp_file_from_ids(datastore_id, image_id)
def test_path_conversion(self, ds_path, expected_os_path):
path = datastore_to_os_path(ds_path)
assert_that(path, equal_to(expected_os_path))
def finalize_image(self, datastore_id, tmp_dir, image_id):
""" Installs an image using image data staged at a temp directory.
"""
self._move_image(image_id, datastore_id, datastore_to_os_path(tmp_dir))
self._create_image_timestamp_file_from_ids(datastore_id, image_id)
