def _delete_unused_images(self, image_sweeper, root):
self._logger.info("IMAGE SCANNER: Sweeper started on %s" % root)
deleted_images = list()
target_images = image_sweeper.get_target_images()
# Compute sweep rest interval
rest_interval_sec = image_sweeper.get_image_sweep_rest_interval()
apply_rate_limiter = False
for curdir, dirs, files in os.walk(root):
# If this contains only other directories skip it
if len(files) == 0:
continue
if apply_rate_limiter:
waste_time(rest_interval_sec)
apply_rate_limiter = True
# On a directory change check if it still needs to run
if image_sweeper.is_stopped():
return
image_id = self._get_and_validate_image_id(curdir, files)
if not image_id:
continue
# If there is not a marker file skip it
marker_pathname = os.path.join(curdir, self.IMAGE_MARKER_FILE_NAME)
if not os.path.isfile(marker_pathname):
continue
# If this is not a part of target images skip it
if image_id not in target_images:
continue
# Write the content of _start_time as an ISO date
# inside the the marker file, any change occurred to
# the image after _start_time invalidates the image
# as a candidate for removal
try:
if self._delete_single_image(image_sweeper, curdir, image_id):
deleted_images.append(image_id)
except Exception as ex:
self._logger.warning("Failed to remove image: %s, %s" %
(curdir, ex))
continue
# Now attempt GCing the image directory.
datastore_id = image_sweeper.datastore_id
try:
self._clean_gc_dir(datastore_id)
except Exception:
# Swallow the exception the next clean call will clear it all.
self._logger.exception("Failed to delete gc dir on datastore %s" %
datastore_id)
return deleted_images
def _collect_active_images(self, image_scanner, root):
"""
:param root: top directory
:return: dictionary of used images, key is image id
"""
# Log messages with prefix: "IMAGE SCANNER" are for debugging
# and will be removed after basic testing
self._logger.info("IMAGE SCANNER: calling collect_active_images()")
# Compute scan rest interval
rest_interval_sec = image_scanner.get_vm_scan_rest_interval()
active_images = dict()
for curdir, dirs, files in os.walk(root):
# On a directory change check if it still needs to run
if image_scanner.is_stopped():
return active_images
# If this contains only other directories skip it
if len(files) == 0:
continue
# Look for the vmdk file
for vm_file in files:
self._logger.info("IMAGE SCANNER: current file %s" % vm_file)
# Skip non vmdk files
if not vm_file.endswith(".vmdk"):
continue
# Skip vmdk delta files
if vm_file.endswith("delta.vmdk"):
continue
# Skip vmdk flat file
if vm_file.endswith("flat.vmdk"):
continue
vmdk_pathname = os.path.join(curdir, vm_file)
self._logger.info("IMAGE SCANNER: found vmdk: %s" %
vmdk_pathname)
try:
vmdk_dictionary = parse_vmdk(vmdk_pathname)
# If there is no file_name_hint, skip it
if image_scanner.FILE_NAME_HINT not in vmdk_dictionary:
# This should be a common occurrence
# the log level should debug
self._logger.info("IMAGE_SCANNER: Vm scan, "
"skipping file: %s "
"missing parent hint" %
vmdk_pathname)
continue
file_name_hint = \
vmdk_dictionary[image_scanner.FILE_NAME_HINT]
image_id = image_scanner.image_manager.\
get_image_id_from_path(file_name_hint)
if image_id not in active_images:
self._logger.info(
"IMAGE SCANNER: adding image_id: %s" % image_id)
active_images[image_id] = file_name_hint
except Exception as ex:
self._logger.warn("Vm scan, skipping file: %s : %s" %
(vmdk_pathname, ex))
waste_time(rest_interval_sec)
return active_images
def _collect_active_images(self, image_scanner, root):
"""
:param root: top directory
:return: dictionary of used images, key is image id
"""
# Log messages with prefix: "IMAGE SCANNER" are for debugging
# and will be removed after basic testing
self._logger.info("IMAGE SCANNER: calling collect_active_images()")
# Compute scan rest interval
rest_interval_sec = image_scanner.get_vm_scan_rest_interval()
active_images = dict()
for curdir, dirs, files in os.walk(root):
# On a directory change check if it still needs to run
if image_scanner.is_stopped():
return active_images
# If this contains only other directories skip it
if len(files) == 0:
continue
# Look for the vmdk file
for vm_file in files:
self._logger.info("IMAGE SCANNER: current file %s" % vm_file)
# Skip non vmdk files
if not vm_file.endswith(".vmdk"):
continue
# Skip vmdk delta files
if vm_file.endswith("delta.vmdk"):
continue
# Skip vmdk flat file
if vm_file.endswith("flat.vmdk"):
continue
vmdk_pathname = os.path.join(curdir, vm_file)
self._logger.info("IMAGE SCANNER: found vmdk: %s"
% vmdk_pathname)
try:
vmdk_dictionary = parse_vmdk(vmdk_pathname)
# If there is no file_name_hint, skip it
if image_scanner.FILE_NAME_HINT not in vmdk_dictionary:
# This should be a common occurrence
# the log level should debug
self._logger.info("IMAGE_SCANNER: Vm scan, "
"skipping file: %s "
"missing parent hint"
% vmdk_pathname)
continue
file_name_hint = \
vmdk_dictionary[image_scanner.FILE_NAME_HINT]
image_id = image_scanner.image_manager.\
get_image_id_from_path(file_name_hint)
if image_id not in active_images:
self._logger.info(
"IMAGE SCANNER: adding image_id: %s" % image_id)
active_images[image_id] = file_name_hint
except Exception as ex:
self._logger.warn("Vm scan, skipping file: %s : %s"
% (vmdk_pathname, ex))
waste_time(rest_interval_sec)
return active_images
def _mark_unused_images(self, image_scanner, root):
self._logger.info("IMAGE SCANNER: Mark unused started on %s" % root)
active_images = image_scanner.get_active_images()
unused_images = dict()
# Compute scan rest interval
rest_interval_sec = image_scanner.get_image_mark_rest_interval()
apply_rate_limiter = False
for curdir, dirs, files in os.walk(root):
# If this contains only other directories skip it
if len(files) == 0 and len(dirs) >= 0:
continue
if apply_rate_limiter:
waste_time(rest_interval_sec)
apply_rate_limiter = True
# On a directory change check if it still needs to run
if image_scanner.is_stopped():
return unused_images
image_id = self._get_and_validate_image_id(curdir, files)
if not image_id:
continue
if image_id in active_images:
self._logger.info(
"IMAGE SCANNER: skipping active image %s" % image_id)
continue
# If there is already a marker file skip it
# but record this image in the unused dictionary
marker_pathname = os.path.join(curdir,
self.IMAGE_MARKER_FILE_NAME)
if os.path.isfile(marker_pathname):
self._logger.info("IMAGE_SCANNER: Adding dir: %s"
% curdir)
unused_images[image_id] = curdir
continue
# Write the content of _start_time to the
# marker file, any change occurred to
# the image after _start_time,
# invalidates the image as a candidate
# for removal
try:
self._write_marker_file(marker_pathname,
image_scanner.start_time_str)
except Exception as ex:
self._logger.warning("Failed to write maker file: %s, %s"
% (marker_pathname, ex))
continue
self._logger.info("IMAGE_SCANNER: Adding dir: %s"
% curdir)
unused_images[image_id] = curdir
return unused_images
def _delete_unused_images(self, image_sweeper, root):
self._logger.info("IMAGE SCANNER: Sweeper started on %s" % root)
deleted_images = list()
target_images = image_sweeper.get_target_images()
# Compute sweep rest interval
rest_interval_sec = image_sweeper.get_image_sweep_rest_interval()
apply_rate_limiter = False
for curdir, dirs, files in os.walk(root):
# If this contains only other directories skip it
if len(files) == 0:
continue
if apply_rate_limiter:
waste_time(rest_interval_sec)
apply_rate_limiter = True
# On a directory change check if it still needs to run
if image_sweeper.is_stopped():
return
image_id = self._get_and_validate_image_id(curdir, files)
if not image_id:
continue
# If there is not a marker file skip it
marker_pathname = os.path.join(curdir, self.IMAGE_MARKER_FILE_NAME)
if not os.path.isfile(marker_pathname):
continue
# If this is not a part of target images skip it
if image_id not in target_images:
continue
# Write the content of _start_time as an ISO date
# inside the the marker file, any change occurred to
# the image after _start_time invalidates the image
# as a candidate for removal
try:
if self._delete_single_image(image_sweeper, curdir, image_id):
deleted_images.append(image_id)
except Exception as ex:
self._logger.warning("Failed to remove image: %s, %s" % (curdir, ex))
continue
# Now attempt GCing the image directory.
datastore_id = image_sweeper.datastore_id
try:
self._clean_gc_dir(datastore_id)
except Exception:
# Swallow the exception the next clean call will clear it all.
self._logger.exception("Failed to delete gc dir on datastore %s" % datastore_id)
return deleted_images
def _mark_unused_images(self, image_scanner, root):
self._logger.info("IMAGE SCANNER: Mark unused started on %s" % root)
active_images = image_scanner.get_active_images()
unused_images = dict()
# Compute scan rest interval
rest_interval_sec = image_scanner.get_image_mark_rest_interval()
apply_rate_limiter = False
for curdir, dirs, files in os.walk(root):
# If this contains only other directories skip it
if len(files) == 0 and len(dirs) >= 0:
continue
if apply_rate_limiter:
waste_time(rest_interval_sec)
apply_rate_limiter = True
# On a directory change check if it still needs to run
if image_scanner.is_stopped():
return unused_images
image_id = self._get_and_validate_image_id(curdir, files)
if not image_id:
continue
if image_id in active_images:
self._logger.info("IMAGE SCANNER: skipping active image %s" %
image_id)
continue
# If there is already a marker file skip it
# but record this image in the unused dictionary
marker_pathname = os.path.join(curdir, self.IMAGE_MARKER_FILE_NAME)
if os.path.isfile(marker_pathname):
self._logger.info("IMAGE_SCANNER: Adding dir: %s" % curdir)
unused_images[image_id] = curdir
continue
# Write the content of _start_time to the
# marker file, any change occurred to
# the image after _start_time,
# invalidates the image as a candidate
# for removal
try:
self._write_marker_file(marker_pathname,
image_scanner.start_time_str)
except Exception as ex:
self._logger.warning("Failed to write maker file: %s, %s" %
(marker_pathname, ex))
continue
self._logger.info("IMAGE_SCANNER: Adding dir: %s" % curdir)
unused_images[image_id] = curdir
return unused_images
def _delete_unused_images(self, image_sweeper, datastore_root):
deleted_images = list()
target_images = image_sweeper.get_target_images()
# Compute sweep rest interval
rest_interval_sec = image_sweeper.get_image_sweep_rest_interval()
for image_id in target_images:
# On a directory change check if it still needs to run
if image_sweeper.is_stopped():
return
image_dir = os.path.join(
datastore_root,
compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id))
# If there is not a marker file, skip it
marker_pathname = os.path.join(
image_dir, self._image_manager.IMAGE_MARKER_FILE_NAME)
if not os.path.isfile(marker_pathname):
self._logger.warn(
"skipping image(%s) because marker file not found" %
image_id)
continue
try:
if self._image_manager._delete_single_image(
image_sweeper, image_dir, image_id):
deleted_images.append(image_id)
except Exception as ex:
self._logger.warning("Failed to remove image: %s, %s" %
(image_dir, ex))
continue
waste_time(rest_interval_sec)
# Now attempt GCing the image directory.
try:
self._image_manager._clean_gc_dir(image_sweeper.datastore_id)
except Exception:
# Swallow the exception the next clean call will clear it all.
self._logger.exception("Failed to delete gc dir on datastore %s" %
image_sweeper.datastore_id)
return deleted_images
def _delete_unused_images(self, image_sweeper):
deleted_images = list()
target_images = image_sweeper.get_target_images()
# Compute sweep rest interval
rest_interval_sec = image_sweeper.get_image_sweep_rest_interval()
for image_id in target_images:
# On a directory change check if it still needs to run
if image_sweeper.is_stopped():
return
image_dir = image_directory_path(image_sweeper.datastore_id, image_id)
# If there is not a marker file, skip it
marker_pathname = os.path.join(image_dir, self._image_manager.IMAGE_MARKER_FILE_NAME)
if not os.path.isfile(marker_pathname):
self._logger.warn("skipping image(%s) because marker file not found" % image_id)
continue
try:
if self._image_manager._delete_single_image(image_sweeper, image_dir, image_id):
deleted_images.append(image_id)
except Exception as ex:
self._logger.warning("Failed to remove image: %s, %s" % (image_dir, ex))
continue
waste_time(rest_interval_sec)
# Now attempt GCing the image directory.
try:
self._image_manager._clean_gc_dir(image_sweeper.datastore_id)
except Exception:
# Swallow the exception the next clean call will clear it all.
self._logger.exception("Failed to delete gc dir on datastore %s" % image_sweeper.datastore_id)
return deleted_images