def _manage_disk(self, op, **kwargs):
try:
self._logger.debug("Invoking %s(%s)" % (op.info.name, kwargs))
task = op(self._manager, **kwargs)
if task:
self._vim_client.wait_for_task(task)
except vim.fault.FileFault, e:
raise DiskFileException(e.msg)
class EsxImageManager(ImageManager):
NUM_MAKEDIRS_ATTEMPTS = 10
DEFAULT_TMP_IMAGES_CLEANUP_INTERVAL = 600.0
IMAGE_TOMBSTONE_FILE_NAME = "image_tombstone.txt"
IMAGE_MARKER_FILE_NAME = "unused_image_marker.txt"
IMAGE_TIMESTAMP_FILE_NAME = "image_timestamp.txt"
IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX = ".renamed"
def __init__(self, vim_client, ds_manager):
super(EsxImageManager, self).__init__()
self._logger = logging.getLogger(__name__)
self._vim_client = vim_client
self._ds_manager = ds_manager
self._image_reaper = None
self._uwsim_nas_exist = None
agent_config = services.get(ServiceName.AGENT_CONFIG)
self._in_uwsim = agent_config.in_uwsim
def monitor_for_cleanup(self,
reap_interval=DEFAULT_TMP_IMAGES_CLEANUP_INTERVAL):
self._image_reaper = Periodic(self.reap_tmp_images, reap_interval)
self._image_reaper.daemon = True
self._image_reaper.start()
def cleanup(self):
if self._image_reaper is not None:
self._image_reaper.stop()
@log_duration
def check_image(self, image_id, datastore):
image_dir = os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME)
try:
return os.path.exists(image_dir)
except:
self._logger.exception("Error looking up %s" % image_dir)
return False
"""
The following method is intended
as a replacement of check_image in
the vm creation workflow compatible
with the new image sweeper.
For an image to be valid both the
directory and the image timestamp
file must exists on the datastore.
"""
def check_and_validate_image(self, image_id, ds_id):
image_dir = os.path.dirname(
os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME))
try:
if not os.path.exists(image_dir):
return False
except:
self._logger.exception("Error looking up %s" % image_dir)
return False
# Check the existence of the timestamp file
timestamp_pathname = \
os.path.join(image_dir,
self.IMAGE_TIMESTAMP_FILE_NAME)
try:
if os.path.exists(timestamp_pathname):
return True
except Exception as ex:
self._logger.exception("Exception looking up %s, %s" %
(timestamp_pathname, ex))
return False
return False
"""
This method is used to update the mod time on the
image timestamp file. It also checks for the existence
of a tombstone file for this image. If the tombstone
file exists it throws an exception.
"""
def touch_image_timestamp(self, ds_id, image_id):
"""
:param ds_id:
:param image_id:
:return:
"""
image_path = os.path.dirname(
os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME))
# Check the existence of the timestamp file
tombstone_pathname = \
os.path.join(image_path,
self.IMAGE_TOMBSTONE_FILE_NAME)
try:
tombstone = os.path.exists(tombstone_pathname)
except Exception as ex:
self._logger.exception("Exception looking up %s, %s" %
(tombstone_pathname, ex))
if tombstone:
raise InvalidImageState
# Touch the timestamp file
timestamp_pathname = \
os.path.join(image_path,
self.IMAGE_TIMESTAMP_FILE_NAME)
try:
os.utime(timestamp_pathname, None)
except Exception as ex:
self._logger.exception("Exception looking up %s, %s" %
(timestamp_pathname, ex))
raise ex
"""
This method is used to create a tombstone marker
in the new image management work flow. The tombstone
marker is a file under the image directory.
"""
def create_image_tombstone(self, ds_id, image_id):
"""
:param ds_id:
:param image_id:
:return:
"""
image_path = os.path.dirname(
os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME))
# Create tombstone file for the image
tombstone_pathname = \
os.path.join(image_path,
self.IMAGE_TOMBSTONE_FILE_NAME)
try:
open(tombstone_pathname, 'w').close()
except Exception as ex:
self._logger.exception("Exception creating %s, %s" %
(tombstone_pathname, ex))
raise ex
self._logger.info("Image: %s tombstoned" % tombstone_pathname)
@log_duration
def check_image_dir(self, image_id, datastore):
image_path = os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME)
try:
return os.path.exists(os.path.dirname(image_path))
except:
self._logger.error("Error looking up %s" % image_path,
exc_info=True)
return False
def get_image_directory_path(self, datastore_id, image_id):
return image_directory_path(datastore_id, image_id)
def get_image_path(self, datastore_id, image_id):
return os_vmdk_path(datastore_id, image_id, IMAGE_FOLDER_NAME)
def image_size(self, image_id):
# TODO(mmutsuzaki) We should iterate over all the image datastores
# until we find one that has the image.
image_ds = list(self._ds_manager.image_datastores())[0]
image_path = os_vmdk_flat_path(image_ds, image_id, IMAGE_FOLDER_NAME)
return os.path.getsize(image_path)
def _load_json(self, metadata_path):
if os.path.exists(metadata_path):
with open(metadata_path) as fh:
try:
data = json.load(fh)
return data
except ValueError:
self._logger.error("Error loading metadata file %s" %
metadata_path,
exc_info=True)
return {}
def get_image_metadata(self, image_id, datastore):
metadata_path = os_metadata_path(datastore, image_id,
IMAGE_FOLDER_NAME)
self._logger.info("Loading metadata %s" % metadata_path)
return self._load_json(metadata_path)
def get_image_manifest(self, image_id):
# This is a shortcut for ttylinux. ttylinux doesn't have manifest file.
if image_id == "ttylinux":
return ImageType.CLOUD, ImageReplication.EAGER
# TODO(mmutsuzaki) We should iterate over all the image datastores
# until we find one that has the image.
image_ds = list(self._ds_manager.image_datastores())[0]
manifest_path = os_image_manifest_path(image_ds, image_id)
if not os.path.isfile(manifest_path):
self._logger.info("Manifest file %s not found" % manifest_path)
return None, None
self._logger.info("Loading manifest %s" % manifest_path)
data = self._load_json(manifest_path)
type = ImageType._NAMES_TO_VALUES[data["imageType"]]
replication = ImageReplication._NAMES_TO_VALUES[
data["imageReplication"]]
return type, replication
def _get_datastore_type(self, datastore_id):
datastores = self._ds_manager.get_datastores()
return [ds.type for ds in datastores if ds.id == datastore_id][0]
def _prepare_virtual_disk_spec(self, disk_type, adapter_type):
"""
:param disk_type [vim.VirtualDiskManager.VirtualDiskType]:
:param adapter_type [vim.VirtualDiskManager.VirtualDiskAdapterType]:
"""
_vd_spec = vim.VirtualDiskManager.VirtualDiskSpec()
_vd_spec.diskType = str(disk_type)
_vd_spec.adapterType = str(adapter_type)
return _vd_spec
def _create_tmp_image(self, source_datastore, source_id, dest_datastore,
dest_id):
""" Copy an image into a temp location.
1. Lock a tmp image destination file with an exclusive lock. This
is to prevent the GC thread from garbage collecting directories
that are actively being used.
The temp directory name contains a random UUID to prevent
collisions with concurrent copies
2. Create the temp directory.
3. Copy the metadata file over.
4. Copy the vmdk over.
@return the tmp image directory on success.
"""
source = vmdk_path(source_datastore, source_id, IMAGE_FOLDER_NAME)
temp_dest = tmp_image_path(dest_datastore, dest_id)
ds_type = self._get_datastore_type(dest_datastore)
tmp_image_dir_path = os.path.dirname(datastore_to_os_path(temp_dest))
# Try grabbing the lock on the temp directory if it fails
# (very unlikely) someone else is copying an image just retry
# later.
with FileBackedLock(tmp_image_dir_path, ds_type):
source_meta = os_metadata_path(source_datastore, source_id,
IMAGE_FOLDER_NAME)
# Create the temp directory
mkdir_p(tmp_image_dir_path)
# Copy the metadata file if it exists.
if os.path.exists(source_meta):
try:
shutil.copy(source_meta, tmp_image_dir_path)
except:
self._logger.exception("Failed to copy metadata file %s",
source_meta)
raise
# Create the timestamp file
self._create_image_timestamp_file(tmp_image_dir_path)
_vd_spec = self._prepare_virtual_disk_spec(
vim.VirtualDiskManager.VirtualDiskType.thin,
vim.VirtualDiskManager.VirtualDiskAdapterType.lsiLogic)
self._manage_disk(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName=source,
destName=temp_dest,
destSpec=_vd_spec)
return tmp_image_dir_path
def _move_image(self, image_id, datastore, tmp_dir):
"""
Atomic move of a tmp folder into the image datastore. Handles
concurrent moves by locking a well know derivative of the image_id
while doing the atomic move.
The exclusive file lock ensures that only one move is successful.
Has the following side effects:
a - If the destination image already exists, it is assumed that
someone else successfully copied the image over and the temp
directory is deleted.
b - If we fail to acquire the file lock after retrying 3 times,
or the atomic move fails, the tmp image directory will be left
behind and needs to be garbage collected later.
image_id: String.The image id of the image being moved.
datastore: String. The datastore id of the datastore.
tmp_dir: String. The absolute path of the temp image directory.
raises: OsError if the move fails
AcquireLockFailure, InvalidFile if we fail to lock the
destination image.
"""
ds_type = self._get_datastore_type(datastore)
image_path = os.path.dirname(
os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME))
parent_path = os.path.dirname(image_path)
# Create the parent image directory if it doesn't exist.
try:
mkdir_p(parent_path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(parent_path):
# Parent directory exists nothing to do.
pass
else:
raise
try:
with FileBackedLock(image_path, ds_type, retry=300,
wait_secs=0.01): # wait lock for 3 seconds
if self._check_image_repair(image_id, datastore):
raise DiskAlreadyExistException("Image already exists")
shutil.move(tmp_dir, image_path)
except (AcquireLockFailure, InvalidFile):
self._logger.info("Unable to lock %s for atomic move" % image_id)
raise
except DiskAlreadyExistException:
self._logger.info("Image %s already copied" % image_id)
rm_rf(tmp_dir)
raise
"""
The following method should be used to check
and validate the existence of a previously
created image. With the new image delete path
the "timestamp" file must exists inside the
image directory. If the directory exists and
the file does not, it may mean that an image
delete operation was aborted mid-way. In this
case the following method recreate the timestamp
file. All operations are performed while
holding the image directory lock (FileBackedLock),
the caller is required to hold the lock.
"""
def _check_image_repair(self, image_id, datastore):
vmdk_pathname = os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME)
image_dirname = os.path.dirname(vmdk_pathname)
try:
# Check vmdk file
if not os.path.exists(vmdk_pathname):
self._logger.info("Vmdk path doesn't exists: %s" %
vmdk_pathname)
return False
except Exception as ex:
self._logger.exception("Exception validating %s, %s" %
(image_dirname, ex))
return False
# Check timestamp file
timestamp_pathname = \
os.path.join(image_dirname,
self.IMAGE_TIMESTAMP_FILE_NAME)
try:
if os.path.exists(timestamp_pathname):
self._logger.info("Timestamp file exists: %s" %
timestamp_pathname)
return True
except Exception as ex:
self._logger.exception("Exception validating %s, %s" %
(timestamp_pathname, ex))
# The timestamp file is not accessible,
# try creating one, if successful try to
# delete the renamed timestamp file if it
# exists
try:
self._create_image_timestamp_file(image_dirname)
self._delete_renamed_image_timestamp_file(image_dirname)
except Exception as ex:
self._logger.exception("Exception creating %s, %s" %
(timestamp_pathname, ex))
return False
self._logger.info("Image repaired: %s" % image_dirname)
return True
def copy_image(self, source_datastore, source_id, dest_datastore, dest_id):
"""Copy an image between datastores.
This method is used to create a "full clone" of a vmdk.
It does so by copying a disk to a unique directory in a well known
temporary directory then moving the disk to the destination image
location. Data in the temporary directory not properly cleaned up
will be periodically garbage collected by the reaper thread.
This minimizes the window during which the vmdk path exists with
incomplete content. It also works around a hostd issue where
cp -f does not work.
The current behavior for when the destination disk exists is
to overwrite said disk.
source_datastore: id of the source datastore
source_id: id of the image to copy from
dest_datastore: id of the destination datastore
dest_id: id of the new image in the destination datastore
throws: AcquireLockFailure if timed out waiting to acquire lock on tmp
image directory
throws: InvalidFile if unable to lock tmp image directory or some other
reasons
"""
if self.check_and_validate_image(dest_id, dest_datastore):
# The image is copied, presumably via some other concurrent
# copy, so we move on.
self._logger.info("Image %s already copied" % dest_id)
raise DiskAlreadyExistException("Image already exists")
# Copy image to the tmp directory.
tmp_dir = self._create_tmp_image(source_datastore, source_id,
dest_datastore, dest_id)
self._move_image(dest_id, dest_datastore, tmp_dir)
def reap_tmp_images(self):
""" Clean up unused directories in the temp image folder. """
for ds in self._ds_manager.get_datastores():
images_dir = tmp_image_folder_os_path(ds.id)
for f in os.listdir(images_dir):
path = os.path.join(images_dir, f)
if not os.path.isdir(path):
continue
try:
with FileBackedLock(path, ds.type):
if (os.path.exists(path)):
self._logger.info("Delete folder %s" % path)
shutil.rmtree(path, ignore_errors=True)
except (AcquireLockFailure, InvalidFile):
self._logger.info("Already locked: %s, skipping" % path)
except:
self._logger.info("Unable to remove %s" % path,
exc_info=True)
def delete_image(self, datastore_id, image_id, ds_type, force):
# Check if the image currently exists
if not self.check_image_dir(image_id, datastore_id):
self._logger.info("Image %s on datastore %s not found" %
(image_id, datastore_id))
raise ImageNotFoundException("Image %s not found" % image_id)
# Mark image as tombstoned
self.create_image_tombstone(datastore_id, image_id)
if not force:
return
# If force try to actively garbage collect the image here
if self._lock_data_disk(datastore_id, image_id):
self._gc_image_dir(datastore_id, image_id)
else:
raise ImageInUse("Image %s is currently in use" % image_id)
# Now attempt GCing the image directory.
try:
self._clean_gc_dir(datastore_id)
except Exception:
# Swallow the exception the next clean call will clear it all.
self._logger.exception("Failed to delete gc dir on datastore %s" %
datastore_id)
def _lock_data_disk(self, datastore_id, image_id):
"""
Lock the data disks associated with the VMs in the provided ref file.
Return True if locking was successful false otherwise.
"""
data_disk = os_vmdk_flat_path(datastore_id, image_id)
try:
# Its ok to delete the data disk as a subsequent power on will
# fail if the data disk is not there.
os.remove(data_disk)
except OSError:
# Remove failed so disk is locked.
self._logger.debug("Disk %s on datastore %s is already locked" %
(data_disk, datastore_id))
return False
return True
def get_images(self, datastore):
""" Get image list from datastore
:param datastore: datastore id
:return: list of string, image id list
"""
image_ids = []
# image_folder is /vmfs/volumes/${datastore}/images
image_folder = os_datastore_path(datastore, IMAGE_FOLDER_NAME)
if not os.path.exists(image_folder):
raise DatastoreNotFoundException()
# prefix is the 2-digit prefix of image id
for prefix in os.listdir(image_folder):
# outer path is something like
# /vmfs/volumes/${datastore}/images/${image_id}[0:2]
outer_path = os.path.join(image_folder, prefix)
if not os.path.isdir(outer_path):
continue
for image_id in os.listdir(outer_path):
if self.check_image(image_id, datastore):
image_ids.append(image_id)
return image_ids
def mark_unused(self, image_scanner):
images_dir_path = os_datastore_path(image_scanner.datastore_id,
IMAGE_FOLDER_NAME)
# Log messages with prefix: "IMAGE SCANNER" are for debugging
# and will be removed after basic testing
self._logger.info("IMAGE SCANNER: images_dir: %s" % images_dir_path)
if not os.path.isdir(images_dir_path):
self._logger.info(
"images_dir_path: images_dir: %s, doesn't exist" %
images_dir_path)
raise DatastoreNotFoundException(
"Image scanner, cannot find image "
"directory for datastore: %s" % image_scanner.datastore_id)
return self._mark_unused_images(image_scanner, images_dir_path)
def delete_unused(self, image_sweeper):
images_dir_path = os_datastore_path(image_sweeper.datastore_id,
IMAGE_FOLDER_NAME)
# Log messages with prefix: "IMAGE SWEEPER" are for debugging
# and will be removed after basic testing
self._logger.info("IMAGE SWEEPER: images_dir: %s" % images_dir_path)
if not os.path.isdir(images_dir_path):
self._logger.info(
"images_dir_path: images_dir: %s, doesn't exist" %
images_dir_path)
raise DatastoreNotFoundException(
"Image sweeper, cannot find image "
"directory for datastore: %s" % image_sweeper.datastore_id)
return self._delete_unused_images(image_sweeper, images_dir_path)
def _unzip(self, src, dst):
self._logger.info("unzip %s -> %s" % (src, dst))
fsrc = gzip.open(src, "rb")
fdst = open(dst, "wb")
try:
shutil.copyfileobj(fsrc, fdst)
finally:
fsrc.close()
fdst.close()
def _copy_disk(self, src, dst):
self._manage_disk(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName=src,
destName=dst)
def _manage_disk(self, op, **kwargs):
if self._in_uwsim:
self._manage_disk_uwsim(op, **kwargs)
return
try:
self._logger.debug("Invoking %s(%s)" % (op.info.name, kwargs))
task = op(self._manager, **kwargs)
self._vim_client.wait_for_task(task)
except vim.Fault.FileAlreadyExists, e:
raise DiskAlreadyExistException(e.msg)
except vim.Fault.FileFault, e:
raise DiskFileException(e.msg)
class EsxImageManager(ImageManager):
NUM_MAKEDIRS_ATTEMPTS = 10
DEFAULT_TMP_IMAGES_CLEANUP_INTERVAL = 600.0
REAP_TMP_IMAGES_GRACE_PERIOD = 600.0
IMAGE_MARKER_FILE_NAME = "unused_image_marker.txt"
IMAGE_TIMESTAMP_FILE_NAME = "image_timestamp.txt"
IMAGE_TIMESTAMP_FILE_RENAME_SUFFIX = ".renamed"
def __init__(self, vim_client, ds_manager):
super(EsxImageManager, self).__init__()
self._logger = logging.getLogger(__name__)
self._vim_client = vim_client
self._ds_manager = ds_manager
self._image_reaper = None
def monitor_for_cleanup(self,
reap_interval=DEFAULT_TMP_IMAGES_CLEANUP_INTERVAL):
self._image_reaper = Periodic(self.reap_tmp_images, reap_interval)
self._image_reaper.daemon = True
self._image_reaper.start()
def cleanup(self):
if self._image_reaper is not None:
self._image_reaper.stop()
@log_duration
def check_image(self, image_id, datastore):
image_dir = os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME_PREFIX)
try:
return os.path.exists(image_dir)
except:
self._logger.exception(
"Error looking up %s" % image_dir)
return False
"""
The following method is intended
as a replacement of check_image in
the vm creation workflow compatible
with the new image sweeper.
For an image to be valid both the
directory and the image timestamp
file must exists on the datastore.
"""
def check_and_validate_image(self, image_id, ds_id):
image_dir = os.path.dirname(
os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME_PREFIX))
try:
if not os.path.exists(image_dir):
return False
except:
self._logger.exception(
"Error looking up %s" % image_dir)
return False
# Check the existence of the timestamp file
timestamp_pathname = \
os.path.join(image_dir,
self.IMAGE_TIMESTAMP_FILE_NAME)
try:
if os.path.exists(timestamp_pathname):
return True
except Exception as ex:
self._logger.exception(
"Exception looking up %s, %s" % (timestamp_pathname, ex))
return False
return False
"""
This method is used to update the mod time on the
image timestamp file.
"""
def touch_image_timestamp(self, ds_id, image_id):
"""
:param ds_id:
:param image_id:
:return:
"""
image_path = os.path.dirname(
os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME_PREFIX))
# Touch the timestamp file
timestamp_pathname = os.path.join(image_path, self.IMAGE_TIMESTAMP_FILE_NAME)
try:
os.utime(timestamp_pathname, None)
except Exception as ex:
self._logger.exception(
"Exception looking up %s, %s" % (timestamp_pathname, ex))
raise ex
@log_duration
def check_image_dir(self, image_id, datastore):
image_path = os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME_PREFIX)
try:
return os.path.exists(os.path.dirname(image_path))
except:
self._logger.error(
"Error looking up %s" % image_path, exc_info=True)
return False
def get_image_directory_path(self, datastore_id, image_id):
return image_directory_path(datastore_id, image_id)
def get_image_path(self, datastore_id, image_id):
return os_vmdk_path(datastore_id, image_id, IMAGE_FOLDER_NAME_PREFIX)
def image_size(self, image_id):
for image_ds in self._ds_manager.image_datastores():
try:
image_path = os_vmdk_flat_path(image_ds, image_id,
IMAGE_FOLDER_NAME_PREFIX)
return os.path.getsize(image_path)
except os.error:
self._logger.info("Image %s not found in DataStore %s" %
(image_id, image_ds))
self._logger.warning("Failed to get image size:",
exc_info=True)
# Failed to access shared image.
raise NoSuchResourceException(
ResourceType.IMAGE,
"Image does not exist.")
def _load_json(self, metadata_path):
if os.path.exists(metadata_path):
with open(metadata_path) as fh:
try:
data = json.load(fh)
return data
except ValueError:
self._logger.error(
"Error loading metadata file %s" % metadata_path,
exc_info=True)
return {}
def get_image_metadata(self, image_id, datastore):
metadata_path = os_metadata_path(datastore,
image_id,
IMAGE_FOLDER_NAME_PREFIX)
self._logger.info("Loading metadata %s" % metadata_path)
return self._load_json(metadata_path)
def _get_datastore_type(self, datastore_id):
datastores = self._ds_manager.get_datastores()
return [ds.type for ds in datastores if ds.id == datastore_id][0]
def _prepare_virtual_disk_spec(self, disk_type, adapter_type):
"""
:param disk_type [vim.VirtualDiskManager.VirtualDiskType]:
:param adapter_type [vim.VirtualDiskManager.VirtualDiskAdapterType]:
"""
_vd_spec = vim.VirtualDiskManager.VirtualDiskSpec()
_vd_spec.diskType = str(disk_type)
_vd_spec.adapterType = str(adapter_type)
return _vd_spec
def _copy_to_tmp_image(self, source_datastore, source_id, dest_datastore, dest_id):
""" Copy an image into a temp location.
1. Lock a tmp image destination file with an exclusive lock. This
is to prevent the GC thread from garbage collecting directories
that are actively being used.
The temp directory name contains a random UUID to prevent
collisions with concurrent copies
2. Create the temp directory.
3. Copy the metadata file over.
4. Copy the vmdk over.
@return the tmp image directory on success.
"""
ds_type = self._get_datastore_type(dest_datastore)
if ds_type == DatastoreType.VSAN:
tmp_image_dir = os_datastore_path(dest_datastore,
compond_path_join(IMAGE_FOLDER_NAME_PREFIX, dest_id),
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
else:
tmp_image_dir = os_datastore_path(dest_datastore,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
# Create the temp directory
self._vim_client.make_directory(tmp_image_dir)
# Copy the metadata file if it exists.
source_meta = os_metadata_path(source_datastore, source_id, IMAGE_FOLDER_NAME_PREFIX)
if os.path.exists(source_meta):
try:
dest_meta = os.path.join(tmp_image_dir, metadata_filename(dest_id))
shutil.copy(source_meta, dest_meta)
except:
self._logger.exception("Failed to copy metadata file %s", source_meta)
raise
# Create the timestamp file
self._create_image_timestamp_file(tmp_image_dir)
_vd_spec = self._prepare_virtual_disk_spec(
vim.VirtualDiskManager.VirtualDiskType.thin,
vim.VirtualDiskManager.VirtualDiskAdapterType.lsiLogic)
self._manage_disk(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName=vmdk_path(source_datastore, source_id, IMAGE_FOLDER_NAME_PREFIX),
destName=os_to_datastore_path(os.path.join(tmp_image_dir, "%s.vmdk" % dest_id)),
destSpec=_vd_spec)
return tmp_image_dir
def _move_image(self, image_id, datastore, tmp_dir):
"""
Atomic move of a tmp folder into the image datastore. Handles
concurrent moves by locking a well know derivative of the image_id
while doing the atomic move.
The exclusive file lock ensures that only one move is successful.
Has the following side effects:
a - If the destination image already exists, it is assumed that
someone else successfully copied the image over and the temp
directory is deleted.
b - If we fail to acquire the file lock after retrying 3 times,
or the atomic move fails, the tmp image directory will be left
behind and needs to be garbage collected later.
image_id: String.The image id of the image being moved.
datastore: String. The datastore id of the datastore.
tmp_dir: String. The absolute path of the temp image directory.
raises: OsError if the move fails
AcquireLockFailure, InvalidFile if we fail to lock the
destination image.
"""
ds_type = self._get_datastore_type(datastore)
image_path = os_datastore_path(datastore, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id))
self._logger.info("_move_image: %s => %s, ds_type: %s" % (tmp_dir, image_path, ds_type))
if not os.path.exists(tmp_dir):
raise ImageNotFoundException("Temp image %s not found" % tmp_dir)
try:
with FileBackedLock(image_path, ds_type, retry=300, wait_secs=0.01): # wait lock for 3 seconds
if self._check_image_repair(image_id, datastore):
raise DiskAlreadyExistException("Image already exists")
if ds_type == DatastoreType.VSAN:
# on VSAN, move all files under [datastore]/image_[image_id]/tmp_image_[uuid]/* to
# [datastore]/image_[image_id]/*.
# Also we do not delete tmp_image folder in success case, because VSAN accesses it
# when creating linked VM, even the folder is now empty.
for entry in os.listdir(tmp_dir):
shutil.move(os.path.join(tmp_dir, entry), os.path.join(image_path, entry))
else:
# on VMFS/NFS/etc, rename [datastore]/tmp_image_[uuid] to [datastore]/tmp_image_[image_id]
self._vim_client.move_file(tmp_dir, image_path)
except:
self._logger.exception("Move image %s to %s failed" % (image_id, image_path))
self._vim_client.delete_file(tmp_dir)
raise
"""
The following method should be used to check
and validate the existence of a previously
created image. With the new image delete path
the "timestamp" file must exists inside the
image directory. If the directory exists and
the file does not, it may mean that an image
delete operation was aborted mid-way. In this
case the following method recreate the timestamp
file. All operations are performed while
holding the image directory lock (FileBackedLock),
the caller is required to hold the lock.
"""
def _check_image_repair(self, image_id, datastore):
vmdk_pathname = os_vmdk_path(datastore,
image_id,
IMAGE_FOLDER_NAME_PREFIX)
image_dirname = os.path.dirname(vmdk_pathname)
try:
# Check vmdk file
if not os.path.exists(vmdk_pathname):
self._logger.info("Vmdk path doesn't exists: %s" %
vmdk_pathname)
return False
except Exception as ex:
self._logger.exception(
"Exception validating %s, %s" % (image_dirname, ex))
return False
# Check timestamp file
timestamp_pathname = \
os.path.join(image_dirname,
self.IMAGE_TIMESTAMP_FILE_NAME)
try:
if os.path.exists(timestamp_pathname):
self._logger.info("Timestamp file exists: %s" %
timestamp_pathname)
return True
except Exception as ex:
self._logger.exception(
"Exception validating %s, %s" % (timestamp_pathname, ex))
# The timestamp file is not accessible,
# try creating one, if successful try to
# delete the renamed timestamp file if it
# exists
try:
self._create_image_timestamp_file(image_dirname)
self._delete_renamed_image_timestamp_file(image_dirname)
except Exception as ex:
self._logger.exception(
"Exception creating %s, %s" % (timestamp_pathname, ex))
return False
self._logger.info("Image repaired: %s" %
image_dirname)
return True
def copy_image(self, source_datastore, source_id, dest_datastore, dest_id):
"""Copy an image between datastores.
This method is used to create a "full clone" of a vmdk.
It does so by copying a disk to a unique directory in a well known
temporary directory then moving the disk to the destination image
location. Data in the temporary directory not properly cleaned up
will be periodically garbage collected by the reaper thread.
This minimizes the window during which the vmdk path exists with
incomplete content. It also works around a hostd issue where
cp -f does not work.
The current behavior for when the destination disk exists is
to overwrite said disk.
source_datastore: id of the source datastore
source_id: id of the image to copy from
dest_datastore: id of the destination datastore
dest_id: id of the new image in the destination datastore
throws: AcquireLockFailure if timed out waiting to acquire lock on tmp
image directory
throws: InvalidFile if unable to lock tmp image directory or some other
reasons
"""
if self.check_and_validate_image(dest_id, dest_datastore):
# The image is copied, presumably via some other concurrent
# copy, so we move on.
self._logger.info("Image %s already copied" % dest_id)
raise DiskAlreadyExistException("Image already exists")
# Copy image to the tmp directory.
tmp_dir = self._copy_to_tmp_image(source_datastore, source_id,
dest_datastore, dest_id)
self._move_image(dest_id, dest_datastore, tmp_dir)
def reap_tmp_images(self):
""" Clean up unused directories in the temp image folder. """
for ds in self._ds_manager.get_datastores():
tmp_image_pattern = os_datastore_path_pattern(ds.id, TMP_IMAGE_FOLDER_NAME_PREFIX)
for image_dir in glob.glob(tmp_image_pattern):
if not os.path.isdir(image_dir):
continue
create_time = os.stat(image_dir).st_ctime
current_time = time.time()
if current_time - self.REAP_TMP_IMAGES_GRACE_PERIOD < create_time:
# Skip folders that are newly created in past x minutes
# For example, during host-to-host transfer, hostd on
# receiving end stores the uploaded file in temp images
# folder but does not lock it with FileBackedLock, so we
# need to allow a grace period before reaping it.
self._logger.info(
"Skip folder: %s, created: %s, now: %s" %
(image_dir, create_time, current_time))
continue
try:
with FileBackedLock(image_dir, ds.type):
if os.path.exists(image_dir):
self._logger.info("Delete folder %s" % image_dir)
shutil.rmtree(image_dir, ignore_errors=True)
except (AcquireLockFailure, InvalidFile):
self._logger.info("Already locked: %s, skipping" % image_dir)
except:
self._logger.info("Unable to remove %s" % image_dir, exc_info=True)
def get_images(self, datastore):
""" Get image list from datastore
:param datastore: datastore id
:return: list of string, image id list
"""
image_ids = []
if not os.path.exists(os_datastore_root(datastore)):
raise DatastoreNotFoundException()
# image_folder is /vmfs/volumes/${datastore}/images_*
image_folder_pattern = os_datastore_path_pattern(datastore,
IMAGE_FOLDER_NAME_PREFIX)
for dir in glob.glob(image_folder_pattern):
image_id = dir.split(COMPOND_PATH_SEPARATOR)[1]
if self.check_image(image_id, datastore):
image_ids.append(image_id)
return image_ids
def _unzip(self, src, dst):
self._logger.info("unzip %s -> %s" % (src, dst))
fsrc = gzip.open(src, "rb")
fdst = open(dst, "wb")
try:
shutil.copyfileobj(fsrc, fdst)
finally:
fsrc.close()
fdst.close()
def _copy_disk(self, src, dst):
self._manage_disk(vim.VirtualDiskManager.CopyVirtualDisk_Task,
sourceName=src, destName=dst)
def _manage_disk(self, op, **kwargs):
try:
self._logger.debug("Invoking %s(%s)" % (op.info.name, kwargs))
task = op(self._manager, **kwargs)
self._vim_client.wait_for_task(task)
except vim.Fault.FileAlreadyExists, e:
raise DiskAlreadyExistException(e.msg)
except vim.Fault.FileFault, e:
raise DiskFileException(e.msg)