def _copy_to_tmp_image(self, source_datastore, source_id, dest_datastore,
dest_id):
""" Copy an image into a temp location.
1. Lock a tmp image destination file with an exclusive lock. This
is to prevent the GC thread from garbage collecting directories
that are actively being used.
The temp directory name contains a random UUID to prevent
collisions with concurrent copies
2. Create the temp directory.
3. Copy the metadata file over.
4. Copy the vmdk over.
@return the tmp image directory on success.
"""
ds_type = self._get_datastore_type(dest_datastore)
if ds_type == DatastoreType.VSAN:
tmp_image_dir = os_datastore_path(
dest_datastore,
compond_path_join(IMAGE_FOLDER_NAME_PREFIX, dest_id),
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX,
str(uuid.uuid4())))
else:
tmp_image_dir = os_datastore_path(
dest_datastore,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX,
str(uuid.uuid4())))
# Create the temp directory
self._host_client.make_directory(tmp_image_dir)
# Copy the metadata file if it exists.
source_meta = os_metadata_path(source_datastore, source_id,
IMAGE_FOLDER_NAME_PREFIX)
if os.path.exists(source_meta):
try:
dest_meta = os.path.join(tmp_image_dir,
metadata_filename(dest_id))
shutil.copy(source_meta, dest_meta)
except:
self._logger.exception("Failed to copy metadata file %s",
source_meta)
raise
# Create the timestamp file
self._create_image_timestamp_file(tmp_image_dir)
self._host_client.copy_disk(
vmdk_path(source_datastore, source_id, IMAGE_FOLDER_NAME_PREFIX),
os.path.join(tmp_image_dir, "%s.vmdk" % dest_id))
return tmp_image_dir
def init_for_create(self, vm_id, datastore, memory, cpus, metadata=None, env=None):
"""Initialize VMConfigSpec for creating a new VM.
"""
vm_path = datastore_path(datastore, compond_path_join(VM_FOLDER_NAME_PREFIX, vm_id))
vm_flags = vim.vm.FlagInfo()
vm_flags.diskUuidEnabled = True
filled_metadata = {}
meta_config = metadata.get("configuration") if metadata else {}
if meta_config:
# The metadata object contains creation configuration details
# that can be augmented by the env map. The only env map entries
# honored are the ones whose key is listed in the "parameters"
# section of the metadata structure.
filled_metadata = meta_config.copy()
param_names = [p["name"] for p in metadata.get("parameters", [])]
if env:
for k, v in env.items():
if k in param_names:
filled_metadata[k] = v
else:
self._logger.warning("Skipped unexpected env: %s" % k)
self._cfg_spec = vim.vm.ConfigSpec()
self._cfg_spec.name = vm_id
self._cfg_spec.guestId = filled_metadata.get('guestOS', 'otherGuest')
self._cfg_spec.memoryMB = memory
self._cfg_spec.numCPUs = cpus
self._cfg_spec.files = vim.vm.FileInfo(vmPathName=vm_path)
self._cfg_spec.deviceChange = []
self._cfg_spec.flags = vm_flags
self._metadata = filled_metadata
def _delete_unused_images(self, image_sweeper, datastore_root):
deleted_images = list()
target_images = image_sweeper.get_target_images()
# Compute sweep rest interval
rest_interval_sec = image_sweeper.get_image_sweep_rest_interval()
for image_id in target_images:
# On a directory change check if it still needs to run
if image_sweeper.is_stopped():
return
image_dir = os.path.join(datastore_root, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id))
# If there is not a marker file, skip it
marker_pathname = os.path.join(image_dir, self._image_manager.UNUSED_IMAGE_MARKER_FILE_NAME)
if not os.path.isfile(marker_pathname):
self._logger.warn("skipping image(%s) because marker file not found" % image_id)
continue
try:
if self._image_manager.delete_image(image_sweeper.datastore_id, image_id,
image_sweeper.get_grace_period()):
deleted_images.append(image_id)
except Exception as ex:
self._logger.warning("Failed to remove image: %s, %s" % (image_dir, ex))
continue
waste_time(rest_interval_sec)
return deleted_images
def init_for_create(self, vm_id, datastore, memory, cpus, metadata=None, env=None):
"""Initialize VMConfigSpec for creating a new VM.
"""
vm_path = datastore_path(datastore, compond_path_join(VM_FOLDER_NAME_PREFIX, vm_id))
vm_flags = vim.vm.FlagInfo()
vm_flags.diskUuidEnabled = True
filled_metadata = {}
meta_config = metadata.get("configuration") if metadata else {}
if meta_config:
# The metadata object contains creation configuration details
# that can be augmented by the env map. The only env map entries
# honored are the ones whose key is listed in the "parameters"
# section of the metadata structure.
filled_metadata = meta_config.copy()
param_names = [p["name"] for p in metadata.get("parameters", [])]
if env:
for k, v in env.items():
if k in param_names:
filled_metadata[k] = v
else:
self._logger.warning("Skipped unexpected env: %s" % k)
self._cfg_spec = vim.vm.ConfigSpec()
self._cfg_spec.name = vm_id
self._cfg_spec.guestId = filled_metadata.get('guestOS', 'otherGuest')
self._cfg_spec.memoryMB = memory
self._cfg_spec.numCPUs = cpus
self._cfg_spec.files = vim.vm.FileInfo(vmPathName=vm_path)
self._cfg_spec.deviceChange = []
self._cfg_spec.flags = vm_flags
self._metadata = filled_metadata
def create_image(self, image_id, datastore_id):
""" Create a temp image on given datastore, return its path.
"""
datastore_type = self._get_datastore_type(datastore_id)
if datastore_type == DatastoreType.VSAN:
# on VSAN, tmp_dir is [datastore]/image_[image_id]/tmp_image_[uuid]
# Because VSAN does not allow moving top-level directories, we place tmp_image
# under image's dir.
relative_path = os.path.join(compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id),
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
tmp_dir = os_datastore_path(datastore_id, relative_path)
else:
# on VMFS/NFS/etc, tmp_dir is [datastore]/tmp_image_[uuid]
tmp_dir = os_datastore_path(datastore_id,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
self._host_client.make_directory(tmp_dir)
# return datastore path, so that it can be passed to nfc client
return os_to_datastore_path(tmp_dir)
def create_image(self, image_id, datastore_id):
""" Create a temp image on given datastore, return its path.
"""
datastore_type = self._get_datastore_type(datastore_id)
if datastore_type == DatastoreType.VSAN:
# on VSAN, tmp_dir is [datastore]/image_[image_id]/tmp_image_[uuid]
# Because VSAN does not allow moving top-level directories, we place tmp_image
# under image's dir.
relative_path = os.path.join(compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id),
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
tmp_dir = os_datastore_path(datastore_id, relative_path)
else:
# on VMFS/NFS/etc, tmp_dir is [datastore]/tmp_image_[uuid]
tmp_dir = os_datastore_path(datastore_id,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
self._host_client.make_directory(tmp_dir)
# return datastore path, so that it can be passed to nfc client
return os_to_datastore_path(tmp_dir)
def _copy_to_tmp_image(self, source_datastore, source_id, dest_datastore, dest_id):
""" Copy an image into a temp location.
1. Lock a tmp image destination file with an exclusive lock. This
is to prevent the GC thread from garbage collecting directories
that are actively being used.
The temp directory name contains a random UUID to prevent
collisions with concurrent copies
2. Create the temp directory.
3. Copy the metadata file over.
4. Copy the vmdk over.
@return the tmp image directory on success.
"""
ds_type = self._get_datastore_type(dest_datastore)
if ds_type == DatastoreType.VSAN:
tmp_image_dir = os_datastore_path(dest_datastore,
compond_path_join(IMAGE_FOLDER_NAME_PREFIX, dest_id),
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
else:
tmp_image_dir = os_datastore_path(dest_datastore,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, str(uuid.uuid4())))
# Create the temp directory
self._host_client.make_directory(tmp_image_dir)
# Copy the metadata file if it exists.
source_meta = os_metadata_path(source_datastore, source_id, IMAGE_FOLDER_NAME_PREFIX)
if os.path.exists(source_meta):
try:
dest_meta = os.path.join(tmp_image_dir, metadata_filename(dest_id))
shutil.copy(source_meta, dest_meta)
except:
self._logger.exception("Failed to copy metadata file %s", source_meta)
raise
# Create the timestamp file
self._create_image_timestamp_file(tmp_image_dir)
self._host_client.copy_disk(vmdk_path(source_datastore, source_id, IMAGE_FOLDER_NAME_PREFIX),
os.path.join(tmp_image_dir, "%s.vmdk" % dest_id))
return tmp_image_dir
def _move_image(self, image_id, datastore, tmp_dir):
"""
Atomic move of a tmp folder into the image datastore. Handles
concurrent moves by locking a well know derivative of the image_id
while doing the atomic move.
The exclusive file lock ensures that only one move is successful.
Has the following side effects:
a - If the destination image already exists, it is assumed that
someone else successfully copied the image over and the temp
directory is deleted.
b - If we fail to acquire the file lock after retrying 3 times,
or the atomic move fails, the tmp image directory will be left
behind and needs to be garbage collected later.
image_id: String.The image id of the image being moved.
datastore: String. The datastore id of the datastore.
tmp_dir: String. The absolute path of the temp image directory.
raises: OsError if the move fails
AcquireLockFailure, InvalidFile if we fail to lock the
destination image.
"""
ds_type = self._get_datastore_type(datastore)
image_path = os_datastore_path(
datastore, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id))
self._logger.info("_move_image: %s => %s, ds_type: %s" %
(tmp_dir, image_path, ds_type))
if not os.path.exists(tmp_dir):
raise ImageNotFoundException("Temp image %s not found" % tmp_dir)
try:
with FileBackedLock(image_path, ds_type, retry=300,
wait_secs=0.1): # wait lock for 30 seconds
if self._check_image_repair(image_id, datastore):
raise DiskAlreadyExistException("Image already exists")
if ds_type == DatastoreType.VSAN:
# on VSAN, move all files under [datastore]/image_[image_id]/tmp_image_[uuid]/* to
# [datastore]/image_[image_id]/*.
# Also we do not delete tmp_image folder in success case, because VSAN accesses it
# when creating linked VM, even the folder is now empty.
for entry in os.listdir(tmp_dir):
shutil.move(os.path.join(tmp_dir, entry),
os.path.join(image_path, entry))
else:
# on VMFS/NFS/etc, rename [datastore]/tmp_image_[uuid] to [datastore]/tmp_image_[image_id]
self._host_client.move_file(tmp_dir, image_path)
except:
self._logger.exception("Move image %s to %s failed" %
(image_id, image_path))
self._host_client.delete_file(tmp_dir)
raise
def _move_image(self, image_id, datastore, tmp_dir):
"""
Atomic move of a tmp folder into the image datastore. Handles
concurrent moves by locking a well know derivative of the image_id
while doing the atomic move.
The exclusive file lock ensures that only one move is successful.
Has the following side effects:
a - If the destination image already exists, it is assumed that
someone else successfully copied the image over and the temp
directory is deleted.
b - If we fail to acquire the file lock after retrying 3 times,
or the atomic move fails, the tmp image directory will be left
behind and needs to be garbage collected later.
image_id: String.The image id of the image being moved.
datastore: String. The datastore id of the datastore.
tmp_dir: String. The absolute path of the temp image directory.
raises: OsError if the move fails
AcquireLockFailure, InvalidFile if we fail to lock the
destination image.
"""
ds_type = self._get_datastore_type(datastore)
image_path = os_datastore_path(datastore, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id))
self._logger.info("_move_image: %s => %s, ds_type: %s" % (tmp_dir, image_path, ds_type))
if not os.path.exists(tmp_dir):
raise ImageNotFoundException("Temp image %s not found" % tmp_dir)
try:
with FileBackedLock(image_path, ds_type, retry=300, wait_secs=0.1): # wait lock for 30 seconds
if self._check_image_repair(image_id, datastore):
raise DiskAlreadyExistException("Image already exists")
if ds_type == DatastoreType.VSAN:
# on VSAN, move all files under [datastore]/image_[image_id]/tmp_image_[uuid]/* to
# [datastore]/image_[image_id]/*.
# Also we do not delete tmp_image folder in success case, because VSAN accesses it
# when creating linked VM, even the folder is now empty.
for entry in os.listdir(tmp_dir):
shutil.move(os.path.join(tmp_dir, entry), os.path.join(image_path, entry))
else:
# on VMFS/NFS/etc, rename [datastore]/tmp_image_[uuid] to [datastore]/tmp_image_[image_id]
self._host_client.move_file(tmp_dir, image_path)
except:
self._logger.exception("Move image %s to %s failed" % (image_id, image_path))
self._host_client.delete_file(tmp_dir)
raise
def get_datastore(self, disk_id):
for datastore in self._ds_manager.get_datastore_ids():
disk = os_vmdk_path(datastore, disk_id)
if os.path.isfile(disk):
return datastore
# Extra logging to help debug failures where host2 cannot find disk created by host1 on a shared datastore
self._logger.error("get_disk_datastore failed: disk=%s, datastores=%s" %
(disk_id, self._ds_manager.get_datastore_ids()))
for datastore in self._ds_manager.get_datastore_ids():
p1 = os_datastore_root(datastore)
p2 = os_datastore_path(datastore, compond_path_join(DISK_FOLDER_NAME_PREFIX, disk_id))
p3 = os_vmdk_path(datastore, disk_id)
self._logger.error("get_disk_datastore check_path: %s:%s, %s:%s, %s:%s" %
(p1, os.path.isdir(p1), p2, os.path.isdir(p2), p3, os.path.isfile(p3)))
return None
def get_datastore(self, disk_id):
for datastore in self._ds_manager.get_datastore_ids():
disk = os_vmdk_path(datastore, disk_id)
if os.path.isfile(disk):
return datastore
# Extra logging to help debug failures where host2 cannot find disk created by host1 on a shared datastore
self._logger.error(
"get_disk_datastore failed: disk=%s, datastores=%s" %
(disk_id, self._ds_manager.get_datastore_ids()))
for datastore in self._ds_manager.get_datastore_ids():
p1 = os_datastore_root(datastore)
p2 = os_datastore_path(
datastore, compond_path_join(DISK_FOLDER_NAME_PREFIX, disk_id))
p3 = os_vmdk_path(datastore, disk_id)
self._logger.error(
"get_disk_datastore check_path: %s:%s, %s:%s, %s:%s" %
(p1, os.path.isdir(p1), p2, os.path.isdir(p2), p3,
os.path.isfile(p3)))
return None
def send_image_to_host(self, source_image_id, source_datastore,
destination_image_id, destination_datastore,
destination_host, destination_port):
self._logger.info("transfer_image: connecting to remote agent")
if self._auth_enabled:
remote_agent_client = DirectClient("Host", Host.Client, destination_host, destination_port, 60,
certfile=SSL_CERT_FILE, keyfile=SSL_KEY_FILE, capath=CA_PATH,
ciphers=SSL_CIPHERS, validate=True)
else:
remote_agent_client = DirectClient("Host", Host.Client, destination_host, destination_port, 60,
validate=False)
remote_agent_client.connect()
self._logger.info("transfer_image: getting ticket")
nfc_ticket = self._get_nfc_ticket(remote_agent_client, destination_datastore)
self._logger.info("transfer_image: creating remote image")
if destination_image_id is None:
destination_image_id = source_image_id
upload_folder = self._create_remote_image(remote_agent_client, destination_image_id, destination_datastore)
try:
source_file_path = datastore_path(source_datastore,
compond_path_join(IMAGE_FOLDER_NAME_PREFIX, source_image_id),
vmdk_add_suffix(source_image_id))
destination_file_path = os.path.join(upload_folder, vmdk_add_suffix(destination_image_id))
self._logger.info("transfer_image: nfc copy image %s => (%s)%s, sslThumbprint=%s, ticket=%s",
source_file_path, destination_host, destination_file_path,
nfc_ticket.ssl_thumbprint, nfc_ticket.session_id)
self._host_client.nfc_copy(source_file_path, destination_host, destination_file_path,
nfc_ticket.ssl_thumbprint, nfc_ticket.session_id)
self._logger.info("transfer_image: finalizing remote image")
self._finalize_remote_image(remote_agent_client, destination_image_id, destination_datastore, upload_folder)
except:
self._logger.info("transfer_image: cleaning up failed transfer")
self._cleanup_remote_image(remote_agent_client, destination_datastore, upload_folder)
raise
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_root,
_uuid):
""" Test that stray images are found and deleted by the reaper """
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_ds_dir = os.path.join(tmpdir, ds.id)
os.mkdir(tmp_ds_dir)
tmp_image_dir = os.path.join(
tmp_ds_dir,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image"))
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_root(datastore):
return os.path.join(tmpdir, datastore)
_os_datastore_root.side_effect = _fake_os_datastore_root
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = ImageManager(self.vim_client, ds_manager)
if not _allow_grace_period:
image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0
time.sleep(0.1)
image_manager.reap_tmp_images()
if _allow_grace_period:
# verify stray image is not deleted due to grace period
self.assertTrue(os.path.exists(path))
else:
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
def _delete_unused_images(self, image_sweeper, datastore_root):
deleted_images = list()
target_images = image_sweeper.get_target_images()
# Compute sweep rest interval
rest_interval_sec = image_sweeper.get_image_sweep_rest_interval()
for image_id in target_images:
# On a directory change check if it still needs to run
if image_sweeper.is_stopped():
return
image_dir = os.path.join(
datastore_root,
compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id))
# If there is not a marker file, skip it
marker_pathname = os.path.join(
image_dir, self._image_manager.UNUSED_IMAGE_MARKER_FILE_NAME)
if not os.path.isfile(marker_pathname):
self._logger.warn(
"skipping image(%s) because marker file not found" %
image_id)
continue
try:
if self._image_manager.delete_image(
image_sweeper.datastore_id, image_id,
image_sweeper.get_grace_period()):
deleted_images.append(image_id)
except Exception as ex:
self._logger.warning("Failed to remove image: %s, %s" %
(image_dir, ex))
continue
waste_time(rest_interval_sec)
return deleted_images
def send_image_to_host(self, source_image_id, source_datastore,
destination_image_id, destination_datastore,
destination_host, destination_port):
self._logger.info("transfer_image: connecting to remote agent")
if self._auth_enabled:
remote_agent_client = DirectClient("Host",
Host.Client,
destination_host,
destination_port,
60,
certfile=SSL_CERT_FILE,
keyfile=SSL_KEY_FILE,
capath=CA_PATH,
ciphers=SSL_CIPHERS,
validate=True)
else:
remote_agent_client = DirectClient("Host",
Host.Client,
destination_host,
destination_port,
60,
validate=False)
remote_agent_client.connect()
self._logger.info("transfer_image: getting ticket")
nfc_ticket = self._get_nfc_ticket(remote_agent_client,
destination_datastore)
self._logger.info("transfer_image: creating remote image")
if destination_image_id is None:
destination_image_id = source_image_id
upload_folder = self._create_remote_image(remote_agent_client,
destination_image_id,
destination_datastore)
try:
source_file_path = datastore_path(
source_datastore,
compond_path_join(IMAGE_FOLDER_NAME_PREFIX, source_image_id),
vmdk_add_suffix(source_image_id))
destination_file_path = os.path.join(
upload_folder, vmdk_add_suffix(destination_image_id))
self._logger.info(
"transfer_image: nfc copy image %s => (%s)%s, sslThumbprint=%s, ticket=%s",
source_file_path, destination_host, destination_file_path,
nfc_ticket.ssl_thumbprint, nfc_ticket.session_id)
self._host_client.nfc_copy(source_file_path, destination_host,
destination_file_path,
nfc_ticket.ssl_thumbprint,
nfc_ticket.session_id)
self._logger.info("transfer_image: finalizing remote image")
self._finalize_remote_image(remote_agent_client,
destination_image_id,
destination_datastore, upload_folder)
except:
self._logger.info("transfer_image: cleaning up failed transfer")
self._cleanup_remote_image(remote_agent_client,
destination_datastore, upload_folder)
raise
def _delete_image(self, image, result_code=DeleteDirectoryResultCode.OK):
resp = self.host_client.delete_directory(
DeleteDirectoryRequest(image.datastore.id, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image.id)))
assert_that(resp.result, is_(result_code))
def create_vm_spec(self, vm_id, datastore, memoryMB, cpus, metadata, env):
vm_path = datastore_path(
datastore, compond_path_join(VM_FOLDER_NAME_PREFIX, vm_id))
spec = self._client.CreateVMSpec(vm_id, vm_path, memoryMB, cpus)
return AttacheVmConfigSpec(self._client, spec)
def create_vm_spec(self, vm_id, datastore, memoryMB, cpus, metadata, env):
vm_path = datastore_path(datastore, compond_path_join(VM_FOLDER_NAME_PREFIX, vm_id))
spec = self._client.CreateVMSpec(vm_id, vm_path, memoryMB, cpus)
return AttacheVmConfigSpec(self._client, spec)
