def create_image_with_vm_disk(self, datastore_id, tmp_dir, image_id,
vm_disk_os_path):
""" Fills a temp image directory with a disk from a VM,
then installs directory in the shared image folder.
"""
dst_vmdk_path = os_datastore_path(datastore_id,
"%s/%s.vmdk" % (tmp_dir, image_id))
if os.path.exists(dst_vmdk_path):
self._logger.warning(
"Unexpected disk %s present, overwriting" % dst_vmdk_path)
dst_vmdk_ds_path = os_to_datastore_path(dst_vmdk_path)
_vd_spec = self._prepare_virtual_disk_spec(
vim.VirtualDiskManager.VirtualDiskType.thin,
vim.VirtualDiskManager.VirtualDiskAdapterType.lsiLogic)
self._manage_disk(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName=os_to_datastore_path(vm_disk_os_path),
destName=dst_vmdk_ds_path,
destSpec=_vd_spec)
try:
self.create_image(datastore_id, tmp_dir, image_id)
except:
self._logger.warning("Delete copied disk %s" % dst_vmdk_ds_path)
self._manage_disk(vim.VirtualDiskManager.DeleteVirtualDisk_Task,
name=dst_vmdk_ds_path)
raise
def get_images(self, datastore):
""" Get image list from datastore
:param datastore: datastore id
:return: list of string, image id list
"""
image_ids = []
# image_folder is /vmfs/volumes/${datastore}/images
image_folder = os_datastore_path(datastore, IMAGE_FOLDER_NAME)
if not os.path.exists(image_folder):
raise DatastoreNotFoundException()
# prefix is the 2-digit prefix of image id
for prefix in os.listdir(image_folder):
# outer path is something like
# /vmfs/volumes/${datastore}/images/${image_id}[0:2]
outer_path = os.path.join(image_folder, prefix)
if not os.path.isdir(outer_path):
continue
for image_id in os.listdir(outer_path):
if self.check_image(image_id, datastore):
image_ids.append(image_id)
return image_ids
def delete_vm(self, vm_id, force=False):
"""Delete a Virtual Machine
:param vm_id: Name of the VM
:type vm_id: str
:param force: Not to check persistent disk, forcefully delete vm.
:type force: boolean
:raise VmPowerStateException when vm is not powered off
"""
vm = self.vim_client.get_vm(vm_id)
if vm.runtime.powerState != 'poweredOff':
raise VmPowerStateException("Can only delete vm in state %s" %
vm.runtime.powerState)
# Getting the path for the new dir structure if we have upgraded from older structure
datastore_name = self.get_vm_datastore(vm.config)
vm_path = os_datastore_path(datastore_name, compond_path_join(VM_FOLDER_NAME_PREFIX, vm_id))
if not force:
self._verify_disks(vm)
self._logger.info("Destroy VM at %s" % vm_path)
self._invoke_vm(vm, "Destroy")
self._ensure_directory_cleanup(vm_path)
self.vim_client.wait_for_vm_delete(vm_id)
def delete_tmp_dir(self, datastore_id, tmp_dir):
""" Deletes a temp image directory by moving it to a GC directory """
file_path = os_datastore_path(datastore_id, tmp_dir)
if not os.path.exists(file_path):
self._logger.info("Tmp dir %s not" % file_path)
raise DirectoryNotFound("Directory %s not found" % file_path)
rm_rf(file_path)
def get_images(self, datastore):
""" Get image list from datastore
:param datastore: datastore id
:return: list of string, image id list
"""
image_ids = []
# image_folder is /vmfs/volumes/${datastore}/images
image_folder = os_datastore_path(datastore, IMAGE_FOLDER_NAME)
if not os.path.exists(image_folder):
raise DatastoreNotFoundException()
# prefix is the 2-digit prefix of image id
for prefix in os.listdir(image_folder):
# outer path is something like
# /vmfs/volumes/${datastore}/images/${image_id}[0:2]
outer_path = os.path.join(image_folder, prefix)
if not os.path.isdir(outer_path):
continue
for image_id in os.listdir(outer_path):
if self.check_image(image_id, datastore):
image_ids.append(image_id)
return image_ids
def _clean_gc_dir(self, datastore_id):
"""
Clean may fail but can be retried later
"""
dir_path = os_datastore_path(datastore_id, GC_IMAGE_FOLDER)
for sub_dir in os.listdir(dir_path):
rm_rf(os.path.join(dir_path, sub_dir))
def _clean_gc_dir(self, datastore_id):
"""
Clean may fail but can be retried later
"""
dir_path = os_datastore_path(datastore_id, GC_IMAGE_FOLDER)
for sub_dir in os.listdir(dir_path):
shutil.rmtree(os.path.join(dir_path, sub_dir), ignore_errors=True)
def create_image_with_vm_disk(self, datastore_id, tmp_dir, image_id,
vm_disk_os_path):
""" Fills a temp image directory with a disk from a VM,
then installs directory in the shared image folder.
"""
dst_vmdk_path = os_datastore_path(datastore_id,
"%s/%s.vmdk" % (tmp_dir, image_id))
if os.path.exists(dst_vmdk_path):
self._logger.warning("Unexpected disk %s present, overwriting" %
dst_vmdk_path)
dst_vmdk_ds_path = os_to_datastore_path(dst_vmdk_path)
_vd_spec = self._prepare_virtual_disk_spec(
vim.VirtualDiskManager.VirtualDiskType.thin,
vim.VirtualDiskManager.VirtualDiskAdapterType.lsiLogic)
self._manage_disk(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName=os_to_datastore_path(vm_disk_os_path),
destName=dst_vmdk_ds_path,
destSpec=_vd_spec)
try:
self.create_image(datastore_id, tmp_dir, image_id)
except:
self._logger.warning("Delete copied disk %s" % dst_vmdk_ds_path)
self._manage_disk(vim.VirtualDiskManager.DeleteVirtualDisk_Task,
name=dst_vmdk_ds_path)
raise
def _clean_gc_dir(self, datastore_id):
"""
Clean may fail but can be retried later
"""
dir_path = os_datastore_path(datastore_id, GC_IMAGE_FOLDER)
for sub_dir in os.listdir(dir_path):
rm_rf(os.path.join(dir_path, sub_dir))
def _clean_gc_dir(self, datastore_id):
"""
Clean may fail but can be retried later
"""
dir_path = os_datastore_path(datastore_id, GC_IMAGE_FOLDER)
for sub_dir in os.listdir(dir_path):
shutil.rmtree(os.path.join(dir_path, sub_dir), ignore_errors=True)
def delete_tmp_dir(self, datastore_id, tmp_dir):
""" Deletes a temp image directory by moving it to a GC directory """
file_path = os_datastore_path(datastore_id, tmp_dir)
if not os.path.exists(file_path):
self._logger.info("Tmp dir %s not" % file_path)
raise DirectoryNotFound("Directory %s not found" % file_path)
rm_rf(file_path)
def _copy_to_tmp_image(self, source_datastore, source_id, dest_datastore, dest_id):
""" Copy an image into a temp location.
1. Lock a tmp image destination file with an exclusive lock. This
is to prevent the GC thread from garbage collecting directories
that are actively being used.
The temp directory name contains a random UUID to prevent
collisions with concurrent copies
2. Create the temp directory.
3. Copy the metadata file over.
4. Copy the vmdk over.
@return the tmp image directory on success.
"""
ds_type = self._get_datastore_type(dest_datastore)
if ds_type == DatastoreType.VSAN:
tmp_image_dir = os_datastore_path(dest_datastore,
compond_path_join(IMAGE_FOLDER_NAME_PREFIX, dest_id),
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
else:
tmp_image_dir = os_datastore_path(dest_datastore,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
# Create the temp directory
self._vim_client.make_directory(tmp_image_dir)
# Copy the metadata file if it exists.
source_meta = os_metadata_path(source_datastore, source_id, IMAGE_FOLDER_NAME_PREFIX)
if os.path.exists(source_meta):
try:
dest_meta = os.path.join(tmp_image_dir, metadata_filename(dest_id))
shutil.copy(source_meta, dest_meta)
except:
self._logger.exception("Failed to copy metadata file %s", source_meta)
raise
# Create the timestamp file
self._create_image_timestamp_file(tmp_image_dir)
_vd_spec = self._prepare_virtual_disk_spec(
vim.VirtualDiskManager.VirtualDiskType.thin,
vim.VirtualDiskManager.VirtualDiskAdapterType.lsiLogic)
self._manage_disk(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName=vmdk_path(source_datastore, source_id, IMAGE_FOLDER_NAME_PREFIX),
destName=os_to_datastore_path(os.path.join(tmp_image_dir, "%s.vmdk" % dest_id)),
destSpec=_vd_spec)
return tmp_image_dir
def _copy_to_tmp_image(self, source_datastore, source_id, dest_datastore, dest_id):
""" Copy an image into a temp location.
1. Lock a tmp image destination file with an exclusive lock. This
is to prevent the GC thread from garbage collecting directories
that are actively being used.
The temp directory name contains a random UUID to prevent
collisions with concurrent copies
2. Create the temp directory.
3. Copy the metadata file over.
4. Copy the vmdk over.
@return the tmp image directory on success.
"""
ds_type = self._get_datastore_type(dest_datastore)
if ds_type == DatastoreType.VSAN:
tmp_image_dir = os_datastore_path(dest_datastore,
compond_path_join(IMAGE_FOLDER_NAME_PREFIX, dest_id),
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
else:
tmp_image_dir = os_datastore_path(dest_datastore,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
# Create the temp directory
self._vim_client.make_directory(tmp_image_dir)
# Copy the metadata file if it exists.
source_meta = os_metadata_path(source_datastore, source_id, IMAGE_FOLDER_NAME_PREFIX)
if os.path.exists(source_meta):
try:
dest_meta = os.path.join(tmp_image_dir, metadata_filename(dest_id))
shutil.copy(source_meta, dest_meta)
except:
self._logger.exception("Failed to copy metadata file %s", source_meta)
raise
# Create the timestamp file
self._create_image_timestamp_file(tmp_image_dir)
_vd_spec = self._prepare_virtual_disk_spec(
vim.VirtualDiskManager.VirtualDiskType.thin,
vim.VirtualDiskManager.VirtualDiskAdapterType.lsiLogic)
self._manage_disk(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName=vmdk_path(source_datastore, source_id, IMAGE_FOLDER_NAME_PREFIX),
destName=os_to_datastore_path(os.path.join(tmp_image_dir, "%s.vmdk" % dest_id)),
destSpec=_vd_spec)
return tmp_image_dir
def create_image(self, image_id, datastore_id):
""" Create a temp image on given datastore, return its path.
"""
datastore_type = self._get_datastore_type(datastore_id)
if datastore_type == DatastoreType.VSAN:
# on VSAN, tmp_dir is [datastore]/image_[image_id]/tmp_image_[uuid]
# Because VSAN does not allow moving top-level directories, we place tmp_image
# under image's dir.
relative_path = os.path.join(compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id),
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
tmp_dir = os_datastore_path(datastore_id, relative_path)
else:
# on VMFS/NFS/etc, tmp_dir is [datastore]/tmp_image_[uuid]
tmp_dir = os_datastore_path(datastore_id,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
self._vim_client.make_directory(tmp_dir)
# return datastore path, so that it can be passed to nfc client
return os_to_datastore_path(tmp_dir)
def create_image(self, image_id, datastore_id):
""" Create a temp image on given datastore, return its path.
"""
datastore_type = self._get_datastore_type(datastore_id)
if datastore_type == DatastoreType.VSAN:
# on VSAN, tmp_dir is [datastore]/image_[image_id]/tmp_image_[uuid]
# Because VSAN does not allow moving top-level directories, we place tmp_image
# under image's dir.
relative_path = os.path.join(compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id),
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
tmp_dir = os_datastore_path(datastore_id, relative_path)
else:
# on VMFS/NFS/etc, tmp_dir is [datastore]/tmp_image_[uuid]
tmp_dir = os_datastore_path(datastore_id,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
self._vim_client.make_directory(tmp_dir)
# return datastore path, so that it can be passed to nfc client
return os_to_datastore_path(tmp_dir)
def delete_unused(self, image_sweeper):
images_dir_path = os_datastore_path(image_sweeper.datastore_id, IMAGE_FOLDER_NAME)
# Log messages with prefix: "IMAGE SWEEPER" are for debugging
# and will be removed after basic testing
self._logger.info("IMAGE SWEEPER: images_dir: %s" % images_dir_path)
if not os.path.isdir(images_dir_path):
self._logger.info("images_dir_path: images_dir: %s, doesn't exist" % images_dir_path)
raise DatastoreNotFoundException(
"Image sweeper, cannot find image " "directory for datastore: %s" % image_sweeper.datastore_id
)
return self._delete_unused_images(image_sweeper, images_dir_path)
def datastore_mkdirs(vim_client, datastore):
for path in [DISK_FOLDER_NAME, VM_FOLDER_NAME, IMAGE_FOLDER_NAME,
TMP_IMAGE_FOLDER_NAME]:
ds_path = os_datastore_path(datastore, path)
# On shared folders os.mkdir races with other hosts attempting to
# create ds_path. try - pass works around that.
try:
os.mkdir(ds_path)
except OSError as e:
if e.errno == errno.EEXIST:
pass
else:
raise
def datastore_mkdirs(vim_client, datastore):
for path in [DISK_FOLDER_NAME, VM_FOLDER_NAME, IMAGE_FOLDER_NAME,
TMP_IMAGE_FOLDER_NAME]:
ds_path = os_datastore_path(datastore, path)
# On shared folders os.mkdir races with other hosts attempting to
# create ds_path. try - pass works around that.
try:
os.mkdir(ds_path)
except OSError as e:
if e.errno == errno.EEXIST:
pass
else:
raise
def send_image_to_host(self, image_id, image_datastore,
destination_image_id, destination_datastore, host,
port):
if destination_image_id is None:
destination_image_id = image_id
metadata = self._read_metadata(image_datastore, image_id)
shadow_vm_id = self._create_shadow_vm()
# place transfer.vmdk under shadow_vm_path to work around VSAN's restriction on
# files at datastore top-level
shadow_vm_path = os_datastore_path(
self._get_shadow_vm_datastore(),
compond_path_join(VM_FOLDER_NAME_PREFIX, shadow_vm_id))
transfer_vmdk_path = os.path.join(shadow_vm_path, "transfer.vmdk")
self._logger.info("transfer_vmdk_path = %s" % transfer_vmdk_path)
agent_client = None
try:
read_lease, disk_url = self._get_image_stream_from_shadow_vm(
image_id, image_datastore, shadow_vm_id)
try:
self.download_file(disk_url, transfer_vmdk_path, read_lease)
finally:
read_lease.Complete()
agent_client = DirectClient("Host", Host.Client, host, port)
agent_client.connect()
vm_path, vm_id = self._prepare_receive_image(
agent_client, destination_image_id, destination_datastore)
spec = self._create_import_vm_spec(vm_id, destination_datastore,
vm_path)
self._send_image(agent_client, host, transfer_vmdk_path, spec)
self._register_imported_image_at_host(agent_client,
destination_image_id,
destination_datastore, vm_id,
metadata)
return vm_id
finally:
try:
os.unlink(transfer_vmdk_path)
except OSError:
pass
self._delete_shadow_vm(shadow_vm_id)
rm_rf(shadow_vm_path)
if agent_client:
agent_client.close()
def get_vm_images(self, image_scanner):
vms_dir_path = os_datastore_path(image_scanner.datastore_id,
VM_FOLDER_NAME)
# Log messages with prefix: "IMAGE SCANNER" are for debugging
# and will be removed after basic testing
self._logger.info("IMAGE SCANNER: vms_dir: %s" % vms_dir_path)
if not os.path.isdir(vms_dir_path):
self._logger.info("get_vm_images: vms_dir: %s, doesn't exist" %
vms_dir_path)
raise DatastoreNotFoundException("Image scanner, cannot find vms "
"directory for datastore: %s" %
image_scanner.datastore_id)
return self._collect_active_images(image_scanner, vms_dir_path)
def send_image_to_host(self, image_id, image_datastore,
destination_image_id, destination_datastore, host,
port):
if destination_image_id is None:
destination_image_id = image_id
metadata = self._read_metadata(image_datastore, image_id)
shadow_vm_id = self._create_shadow_vm()
tmp_path = "/vmfs/volumes/%s/%s_transfer.vmdk" % (
self._get_shadow_vm_datastore(), shadow_vm_id)
self._logger.info("http_disk_transfer: tmp_path = %s" % tmp_path)
agent_client = None
try:
read_lease, disk_url = self._get_image_stream_from_shadow_vm(
image_id, image_datastore, shadow_vm_id)
try:
self.download_file(disk_url, tmp_path, read_lease)
finally:
read_lease.Complete()
agent_client = DirectClient("Host", Host.Client, host, port)
agent_client.connect()
vm_path, vm_id = self._prepare_receive_image(
agent_client, destination_image_id, destination_datastore)
spec = self._create_import_vm_spec(vm_id, destination_datastore,
vm_path)
self._send_image(agent_client, host, tmp_path, spec)
self._register_imported_image_at_host(agent_client,
destination_image_id,
destination_datastore, vm_id,
metadata)
return vm_id
finally:
try:
os.unlink(tmp_path)
except OSError:
pass
self._delete_shadow_vm(shadow_vm_id)
rm_rf(
os_datastore_path(
self._get_shadow_vm_datastore(),
compond_path_join(VM_FOLDER_NAME_PREFIX, shadow_vm_id)))
if agent_client:
agent_client.close()
def delete_unused(self, image_sweeper):
images_dir_path = os_datastore_path(image_sweeper.datastore_id,
IMAGE_FOLDER_NAME)
# Log messages with prefix: "IMAGE SWEEPER" are for debugging
# and will be removed after basic testing
self._logger.info("IMAGE SWEEPER: images_dir: %s" % images_dir_path)
if not os.path.isdir(images_dir_path):
self._logger.info(
"images_dir_path: images_dir: %s, doesn't exist" %
images_dir_path)
raise DatastoreNotFoundException(
"Image sweeper, cannot find image "
"directory for datastore: %s" % image_sweeper.datastore_id)
return self._delete_unused_images(image_sweeper, images_dir_path)
def get_vm_images(self, image_scanner):
vms_dir_path = os_datastore_path(image_scanner.datastore_id,
VM_FOLDER_NAME)
# Log messages with prefix: "IMAGE SCANNER" are for debugging
# and will be removed after basic testing
self._logger.info("IMAGE SCANNER: vms_dir: %s" % vms_dir_path)
if not os.path.isdir(vms_dir_path):
self._logger.info("get_vm_images: vms_dir: %s, doesn't exist"
% vms_dir_path)
raise DatastoreNotFoundException(
"Image scanner, cannot find vms "
"directory for datastore: %s"
% image_scanner.datastore_id)
return self._collect_active_images(image_scanner, vms_dir_path)
def create_image(self, datastore_id, tmp_dir, image_id):
""" Installs an image using image data staged at a temp directory.
"""
src_path = os_datastore_path(datastore_id, tmp_dir)
if not os.path.exists(src_path):
self._logger.info("Tmp dir %s on datastore %s not found" % (tmp_dir, datastore_id))
raise ImageNotFoundException("Image %s not found" % src_path)
# Check if the dest image id already exists
if self.check_image_dir(image_id, datastore_id):
self._logger.info("Image %s on datastore %s already exists" % (image_id, datastore_id))
raise DiskAlreadyExistException()
self._move_image(image_id, datastore_id, src_path)
self._create_image_timestamp_file_from_ids(datastore_id, image_id)
def _move_image(self, image_id, datastore, tmp_dir):
"""
Atomic move of a tmp folder into the image datastore. Handles
concurrent moves by locking a well know derivative of the image_id
while doing the atomic move.
The exclusive file lock ensures that only one move is successful.
Has the following side effects:
a - If the destination image already exists, it is assumed that
someone else successfully copied the image over and the temp
directory is deleted.
b - If we fail to acquire the file lock after retrying 3 times,
or the atomic move fails, the tmp image directory will be left
behind and needs to be garbage collected later.
image_id: String.The image id of the image being moved.
datastore: String. The datastore id of the datastore.
tmp_dir: String. The absolute path of the temp image directory.
raises: OsError if the move fails
AcquireLockFailure, InvalidFile if we fail to lock the
destination image.
"""
ds_type = self._get_datastore_type(datastore)
image_path = os_datastore_path(datastore, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id))
self._logger.info("_move_image: %s => %s, ds_type: %s" % (tmp_dir, image_path, ds_type))
if not os.path.exists(tmp_dir):
raise ImageNotFoundException("Temp image %s not found" % tmp_dir)
try:
with FileBackedLock(image_path, ds_type, retry=300, wait_secs=0.01): # wait lock for 3 seconds
if self._check_image_repair(image_id, datastore):
raise DiskAlreadyExistException("Image already exists")
if ds_type == DatastoreType.VSAN:
# on VSAN, move all files under [datastore]/image_[image_id]/tmp_image_[uuid]/* to
# [datastore]/image_[image_id]/*.
# Also we do not delete tmp_image folder in success case, because VSAN accesses it
# when creating linked VM, even the folder is now empty.
for entry in os.listdir(tmp_dir):
shutil.move(os.path.join(tmp_dir, entry), os.path.join(image_path, entry))
else:
# on VMFS/NFS/etc, rename [datastore]/tmp_image_[uuid] to [datastore]/tmp_image_[image_id]
self._vim_client.move_file(tmp_dir, image_path)
except:
self._logger.exception("Move image %s to %s failed" % (image_id, image_path))
self._vim_client.delete_file(tmp_dir)
raise
def _move_image(self, image_id, datastore, tmp_dir):
"""
Atomic move of a tmp folder into the image datastore. Handles
concurrent moves by locking a well know derivative of the image_id
while doing the atomic move.
The exclusive file lock ensures that only one move is successful.
Has the following side effects:
a - If the destination image already exists, it is assumed that
someone else successfully copied the image over and the temp
directory is deleted.
b - If we fail to acquire the file lock after retrying 3 times,
or the atomic move fails, the tmp image directory will be left
behind and needs to be garbage collected later.
image_id: String.The image id of the image being moved.
datastore: String. The datastore id of the datastore.
tmp_dir: String. The absolute path of the temp image directory.
raises: OsError if the move fails
AcquireLockFailure, InvalidFile if we fail to lock the
destination image.
"""
ds_type = self._get_datastore_type(datastore)
image_path = os_datastore_path(datastore, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id))
self._logger.info("_move_image: %s => %s, ds_type: %s" % (tmp_dir, image_path, ds_type))
if not os.path.exists(tmp_dir):
raise ImageNotFoundException("Temp image %s not found" % tmp_dir)
try:
with FileBackedLock(image_path, ds_type, retry=300, wait_secs=0.01): # wait lock for 3 seconds
if self._check_image_repair(image_id, datastore):
raise DiskAlreadyExistException("Image already exists")
if ds_type == DatastoreType.VSAN:
# on VSAN, move all files under [datastore]/image_[image_id]/tmp_image_[uuid]/* to
# [datastore]/image_[image_id]/*.
# Also we do not delete tmp_image folder in success case, because VSAN accesses it
# when creating linked VM, even the folder is now empty.
for entry in os.listdir(tmp_dir):
shutil.move(os.path.join(tmp_dir, entry), os.path.join(image_path, entry))
else:
# on VMFS/NFS/etc, rename [datastore]/tmp_image_[uuid] to [datastore]/tmp_image_[image_id]
self._vim_client.move_file(tmp_dir, image_path)
except:
self._logger.exception("Move image %s to %s failed" % (image_id, image_path))
self._vim_client.delete_file(tmp_dir)
raise
def create_image(self, datastore_id, tmp_dir, image_id):
""" Installs an image using image data staged at a temp directory.
"""
src_path = os_datastore_path(datastore_id, tmp_dir)
if not os.path.exists(src_path):
self._logger.info("Tmp dir %s on datastore %s not found" %
(tmp_dir, datastore_id))
raise ImageNotFoundException("Image %s not found" % src_path)
# Check if the dest image id already exists
if self.check_image_dir(image_id, datastore_id):
self._logger.info("Image %s on datastore %s already exists" %
(image_id, datastore_id))
raise DiskAlreadyExistException()
self._move_image(image_id, datastore_id, src_path)
self._create_image_timestamp_file_from_ids(datastore_id, image_id)
def send_image_to_host(self, image_id, image_datastore,
destination_image_id, destination_datastore,
host, port):
manifest, metadata = self._read_metadata(image_datastore, image_id)
shadow_vm_id = self._create_shadow_vm()
tmp_path = "/vmfs/volumes/%s/%s_transfer.vmdk" % (
self._get_shadow_vm_datastore(), shadow_vm_id)
try:
read_lease, disk_url =\
self._get_image_stream_from_shadow_vm(
image_id, image_datastore, shadow_vm_id)
try:
self.download_file(disk_url, tmp_path, read_lease)
finally:
read_lease.Complete()
if destination_image_id is None:
destination_image_id = image_id
spec = self._create_import_vm_spec(
destination_image_id, destination_datastore)
imported_vm_name = self._send_image(
tmp_path, manifest, metadata, spec, destination_image_id,
destination_datastore, host, port)
return imported_vm_name
finally:
try:
os.unlink(tmp_path)
except OSError:
pass
self._delete_shadow_vm(shadow_vm_id)
rm_rf(os_datastore_path(self._get_shadow_vm_datastore(),
compond_path_join(VM_FOLDER_NAME_PREFIX, shadow_vm_id)))
def _gc_image_dir(self, datastore_id, image_id):
"""
Moves the current image directory into the GC image folder and
relies on the later GC call for cleanup.
Exception is thrown if the move fails.
Assumes the ref files contained in the image director are locked, so
there can only be one move happening at any point in time.
"""
src_path = os.path.dirname(os_vmdk_path(datastore_id, image_id, IMAGE_FOLDER_NAME))
# Verify locking held.
assert os.path.exists(src_path)
# Generate a random suffix, this is to address the following.
# a - Image disk say i1 was moved to the gc_dir
# b - Cleanup of i1 failed for whatever reason.
# c - New copy of i1 was uploaded to the datastore.
# d - The new copy is now being deleted.
# The location of the move in d needs to be different from the
# location of the move from b, hence use a random uuid.
rnd_uuid = str(uuid.uuid4())
gc_dir = os_datastore_path(datastore_id, GC_IMAGE_FOLDER)
dst_dir = os.path.join(gc_dir, rnd_uuid)
os.makedirs(dst_dir)
shutil.move(src_path, dst_dir)
def _gc_image_dir(self, datastore_id, image_id):
"""
Moves the current image directory into the GC image folder and
relies on the later GC call for cleanup.
Exception is thrown if the move fails.
Assumes the ref files contained in the image director are locked, so
there can only be one move happening at any point in time.
"""
src_path = os.path.dirname(
os_vmdk_path(datastore_id, image_id, IMAGE_FOLDER_NAME))
# Verify locking held.
assert (os.path.exists(src_path))
# Generate a random suffix, this is to address the following.
# a - Image disk say i1 was moved to the gc_dir
# b - Cleanup of i1 failed for whatever reason.
# c - New copy of i1 was uploaded to the datastore.
# d - The new copy is now being deleted.
# The location of the move in d needs to be different from the
# location of the move from b, hence use a random uuid.
rnd_uuid = str(uuid.uuid4())
gc_dir = os_datastore_path(datastore_id, GC_IMAGE_FOLDER)
dst_dir = os.path.join(gc_dir, rnd_uuid)
os.makedirs(dst_dir)
shutil.move(src_path, dst_dir)
def _do_delete_single_image(self, image_sweeper,
curdir, image_id,
modify):
self._logger.info("IMAGE SCANNER: Starting to "
"delete image: %s, %s"
% (curdir, image_id))
# Read content of marker file
try:
marker_pathname = os.path.join(curdir,
self.IMAGE_MARKER_FILE_NAME)
marker_time = self._read_marker_file(marker_pathname)
except Exception as ex:
self._logger.warning("Cannot read marker file: %s, %s"
% (curdir, ex))
return False
self._logger.info("IMAGE SCANNER: Marker time: %s"
% marker_time)
# Subtract grace time to avoid
# errors due to small difference in clock
# values on different hosts. Pretend the scan
# started 60 seconds earlier.
marker_time -= image_sweeper.get_grace_period()
self._logger.info(
"IMAGE SCANNER: Marker time after grace: %s"
% marker_time)
timestamp_pathname = \
os.path.join(curdir,
self.IMAGE_TIMESTAMP_FILE_NAME)
renamed_timestamp_pathname = \
timestamp_pathname + \
self.IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX
# Lock image
datastore_id = image_sweeper.datastore_id
ds_type = self._get_datastore_type(datastore_id)
with FileBackedLock(curdir, ds_type):
# Get mod time of the timestamp file,
# the method returns None if the file doesn't
# exists, throws exception if there are
# other errors
timestamp_exists, mod_time = \
self._get_mod_time(
timestamp_pathname)
if timestamp_exists:
# Marker time is out of date
# skip this image
self._logger.info(
"IMAGE SCANNER: mod time: %s"
% mod_time)
if mod_time >= marker_time:
# Remove marker file
self._logger.info(
"IMAGE SCANNER: mod time too recent")
self._image_sweeper_unlink(marker_pathname)
return False
# Move timestamp file to a new name
if modify:
self._image_sweeper_rename(
timestamp_pathname,
renamed_timestamp_pathname)
else:
# If we could not find the timestamp file
# it may mean that this was a partially
# removed image, log message and continue
self._logger.info("Cannot find timestamp file: %s"
"continuing with image removal"
% timestamp_pathname)
if modify:
# Get mod time of the renamed timestamp file
renamed_timestamp_exists, renamed_mod_time = \
self._get_mod_time(
renamed_timestamp_pathname)
else:
renamed_timestamp_exists = True
renamed_mod_time = mod_time
self._logger.info(
"IMAGE SCANNER: rename timestamp exists: %s, "
"renamed mod time: %s" %
(renamed_timestamp_exists, renamed_mod_time))
# If there was timestamp file but there
# is no renamed-timestamp file something
# bad might have happened, skip this image
if timestamp_exists and \
not renamed_timestamp_exists:
self._logger.warning("Error, missing renamed "
"timestamp file: %s"
% renamed_timestamp_pathname)
return False
# Normal case both timestamp and renamed
# timestamp exist
if timestamp_exists and renamed_timestamp_exists:
# Normal case: both timestamp and renamed
# timestamp files exist. If the mod time on the
# renamed-timestamp has changed skip this image.
if renamed_mod_time != mod_time:
self._logger.info("mod time changed on renamed "
"timestamp file, %s: %d -> %d" %
(renamed_timestamp_pathname,
mod_time, renamed_mod_time))
self._image_sweeper_unlink(marker_pathname)
return False
elif renamed_timestamp_exists:
# Only the renamed timestamp file exists
# Check the mod time of the renamed-timestamp
# file against the marker time
if renamed_mod_time >= marker_time:
self._image_sweeper_unlink(marker_pathname)
return False
# Move directory
self._logger.info("IMAGE SCANNER: removing image: %s"
% curdir)
if modify:
trash_dir = os.path.join(
os_datastore_path(datastore_id, GC_IMAGE_FOLDER),
image_id)
self._image_sweeper_rename(curdir, trash_dir)
# Unlock
# Delete image
if modify:
self._image_sweeper_rm_rf(trash_dir)
return True
def _do_delete_single_image(self, image_sweeper, curdir, image_id, modify):
self._logger.info("IMAGE SCANNER: Starting to "
"delete image: %s, %s" % (curdir, image_id))
# Read content of marker file
try:
marker_pathname = os.path.join(curdir, self.IMAGE_MARKER_FILE_NAME)
marker_time = self._read_marker_file(marker_pathname)
except Exception as ex:
self._logger.warning("Cannot read marker file: %s, %s" %
(curdir, ex))
return False
self._logger.info("IMAGE SCANNER: Marker time: %s" % marker_time)
# Subtract grace time to avoid
# errors due to small difference in clock
# values on different hosts. Pretend the scan
# started 60 seconds earlier.
marker_time -= image_sweeper.get_grace_period()
self._logger.info("IMAGE SCANNER: Marker time after grace: %s" %
marker_time)
timestamp_pathname = \
os.path.join(curdir,
self.IMAGE_TIMESTAMP_FILE_NAME)
renamed_timestamp_pathname = \
timestamp_pathname + \
self.IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX
# Lock image
datastore_id = image_sweeper.datastore_id
ds_type = self._get_datastore_type(datastore_id)
with FileBackedLock(curdir, ds_type):
# Get mod time of the timestamp file,
# the method returns None if the file doesn't
# exists, throws exception if there are
# other errors
timestamp_exists, mod_time = \
self._get_mod_time(
timestamp_pathname)
if timestamp_exists:
# Marker time is out of date
# skip this image
self._logger.info("IMAGE SCANNER: mod time: %s" % mod_time)
if mod_time >= marker_time:
# Remove marker file
self._logger.info("IMAGE SCANNER: mod time too recent")
self._image_sweeper_unlink(marker_pathname)
return False
# Move timestamp file to a new name
if modify:
self._image_sweeper_rename(timestamp_pathname,
renamed_timestamp_pathname)
else:
# If we could not find the timestamp file
# it may mean that this was a partially
# removed image, log message and continue
self._logger.info("Cannot find timestamp file: %s"
"continuing with image removal" %
timestamp_pathname)
if modify:
# Get mod time of the renamed timestamp file
renamed_timestamp_exists, renamed_mod_time = \
self._get_mod_time(
renamed_timestamp_pathname)
else:
renamed_timestamp_exists = True
renamed_mod_time = mod_time
self._logger.info("IMAGE SCANNER: rename timestamp exists: %s, "
"renamed mod time: %s" %
(renamed_timestamp_exists, renamed_mod_time))
# If there was timestamp file but there
# is no renamed-timestamp file something
# bad might have happened, skip this image
if timestamp_exists and \
not renamed_timestamp_exists:
self._logger.warning("Error, missing renamed "
"timestamp file: %s" %
renamed_timestamp_pathname)
return False
# Normal case both timestamp and renamed
# timestamp exist
if timestamp_exists and renamed_timestamp_exists:
# Normal case: both timestamp and renamed
# timestamp files exist. If the mod time on the
# renamed-timestamp has changed skip this image.
if renamed_mod_time != mod_time:
self._logger.info("mod time changed on renamed "
"timestamp file, %s: %d -> %d" %
(renamed_timestamp_pathname, mod_time,
renamed_mod_time))
self._image_sweeper_unlink(marker_pathname)
return False
elif renamed_timestamp_exists:
# Only the renamed timestamp file exists
# Check the mod time of the renamed-timestamp
# file against the marker time
if renamed_mod_time >= marker_time:
self._image_sweeper_unlink(marker_pathname)
return False
# Move directory
self._logger.info("IMAGE SCANNER: removing image: %s" % curdir)
if modify:
trash_dir = os.path.join(
os_datastore_path(datastore_id, GC_IMAGE_FOLDER), image_id)
self._image_sweeper_rename(curdir, trash_dir)
# Unlock
# Delete image
if modify:
self._image_sweeper_rm_rf(trash_dir)
return True
