def _delete_unused_images(self, image_sweeper, root): self._logger.info("IMAGE SCANNER: Sweeper started on %s" % root) deleted_images = list() target_images = image_sweeper.get_target_images() # Compute sweep rest interval rest_interval_sec = image_sweeper.get_image_sweep_rest_interval() apply_rate_limiter = False for curdir, dirs, files in os.walk(root): # If this contains only other directories skip it if len(files) == 0: continue if apply_rate_limiter: waste_time(rest_interval_sec) apply_rate_limiter = True # On a directory change check if it still needs to run if image_sweeper.is_stopped(): return image_id = self._get_and_validate_image_id(curdir, files) if not image_id: continue # If there is not a marker file skip it marker_pathname = os.path.join(curdir, self.IMAGE_MARKER_FILE_NAME) if not os.path.isfile(marker_pathname): continue # If this is not a part of target images skip it if image_id not in target_images: continue # Write the content of _start_time as an ISO date # inside the the marker file, any change occurred to # the image after _start_time invalidates the image # as a candidate for removal try: if self._delete_single_image(image_sweeper, curdir, image_id): deleted_images.append(image_id) except Exception as ex: self._logger.warning("Failed to remove image: %s, %s" % (curdir, ex)) continue # Now attempt GCing the image directory. datastore_id = image_sweeper.datastore_id try: self._clean_gc_dir(datastore_id) except Exception: # Swallow the exception the next clean call will clear it all. self._logger.exception("Failed to delete gc dir on datastore %s" % datastore_id) return deleted_images
def _collect_active_images(self, image_scanner, root): """ :param root: top directory :return: dictionary of used images, key is image id """ # Log messages with prefix: "IMAGE SCANNER" are for debugging # and will be removed after basic testing self._logger.info("IMAGE SCANNER: calling collect_active_images()") # Compute scan rest interval rest_interval_sec = image_scanner.get_vm_scan_rest_interval() active_images = dict() for curdir, dirs, files in os.walk(root): # On a directory change check if it still needs to run if image_scanner.is_stopped(): return active_images # If this contains only other directories skip it if len(files) == 0: continue # Look for the vmdk file for vm_file in files: self._logger.info("IMAGE SCANNER: current file %s" % vm_file) # Skip non vmdk files if not vm_file.endswith(".vmdk"): continue # Skip vmdk delta files if vm_file.endswith("delta.vmdk"): continue # Skip vmdk flat file if vm_file.endswith("flat.vmdk"): continue vmdk_pathname = os.path.join(curdir, vm_file) self._logger.info("IMAGE SCANNER: found vmdk: %s" % vmdk_pathname) try: vmdk_dictionary = parse_vmdk(vmdk_pathname) # If there is no file_name_hint, skip it if image_scanner.FILE_NAME_HINT not in vmdk_dictionary: # This should be a common occurrence # the log level should debug self._logger.info("IMAGE_SCANNER: Vm scan, " "skipping file: %s " "missing parent hint" % vmdk_pathname) continue file_name_hint = \ vmdk_dictionary[image_scanner.FILE_NAME_HINT] image_id = image_scanner.image_manager.\ get_image_id_from_path(file_name_hint) if image_id not in active_images: self._logger.info( "IMAGE SCANNER: adding image_id: %s" % image_id) active_images[image_id] = file_name_hint except Exception as ex: self._logger.warn("Vm scan, skipping file: %s : %s" % (vmdk_pathname, ex)) waste_time(rest_interval_sec) return active_images
def _mark_unused_images(self, image_scanner, root): self._logger.info("IMAGE SCANNER: Mark unused started on %s" % root) active_images = image_scanner.get_active_images() unused_images = dict() # Compute scan rest interval rest_interval_sec = image_scanner.get_image_mark_rest_interval() apply_rate_limiter = False for curdir, dirs, files in os.walk(root): # If this contains only other directories skip it if len(files) == 0 and len(dirs) >= 0: continue if apply_rate_limiter: waste_time(rest_interval_sec) apply_rate_limiter = True # On a directory change check if it still needs to run if image_scanner.is_stopped(): return unused_images image_id = self._get_and_validate_image_id(curdir, files) if not image_id: continue if image_id in active_images: self._logger.info( "IMAGE SCANNER: skipping active image %s" % image_id) continue # If there is already a marker file skip it # but record this image in the unused dictionary marker_pathname = os.path.join(curdir, self.IMAGE_MARKER_FILE_NAME) if os.path.isfile(marker_pathname): self._logger.info("IMAGE_SCANNER: Adding dir: %s" % curdir) unused_images[image_id] = curdir continue # Write the content of _start_time to the # marker file, any change occurred to # the image after _start_time, # invalidates the image as a candidate # for removal try: self._write_marker_file(marker_pathname, image_scanner.start_time_str) except Exception as ex: self._logger.warning("Failed to write maker file: %s, %s" % (marker_pathname, ex)) continue self._logger.info("IMAGE_SCANNER: Adding dir: %s" % curdir) unused_images[image_id] = curdir return unused_images
def _mark_unused_images(self, image_scanner, root): self._logger.info("IMAGE SCANNER: Mark unused started on %s" % root) active_images = image_scanner.get_active_images() unused_images = dict() # Compute scan rest interval rest_interval_sec = image_scanner.get_image_mark_rest_interval() apply_rate_limiter = False for curdir, dirs, files in os.walk(root): # If this contains only other directories skip it if len(files) == 0 and len(dirs) >= 0: continue if apply_rate_limiter: waste_time(rest_interval_sec) apply_rate_limiter = True # On a directory change check if it still needs to run if image_scanner.is_stopped(): return unused_images image_id = self._get_and_validate_image_id(curdir, files) if not image_id: continue if image_id in active_images: self._logger.info("IMAGE SCANNER: skipping active image %s" % image_id) continue # If there is already a marker file skip it # but record this image in the unused dictionary marker_pathname = os.path.join(curdir, self.IMAGE_MARKER_FILE_NAME) if os.path.isfile(marker_pathname): self._logger.info("IMAGE_SCANNER: Adding dir: %s" % curdir) unused_images[image_id] = curdir continue # Write the content of _start_time to the # marker file, any change occurred to # the image after _start_time, # invalidates the image as a candidate # for removal try: self._write_marker_file(marker_pathname, image_scanner.start_time_str) except Exception as ex: self._logger.warning("Failed to write maker file: %s, %s" % (marker_pathname, ex)) continue self._logger.info("IMAGE_SCANNER: Adding dir: %s" % curdir) unused_images[image_id] = curdir return unused_images
def _delete_unused_images(self, image_sweeper, datastore_root): deleted_images = list() target_images = image_sweeper.get_target_images() # Compute sweep rest interval rest_interval_sec = image_sweeper.get_image_sweep_rest_interval() for image_id in target_images: # On a directory change check if it still needs to run if image_sweeper.is_stopped(): return image_dir = os.path.join( datastore_root, compond_path_join(IMAGE_FOLDER_NAME_PREFIX, image_id)) # If there is not a marker file, skip it marker_pathname = os.path.join( image_dir, self._image_manager.IMAGE_MARKER_FILE_NAME) if not os.path.isfile(marker_pathname): self._logger.warn( "skipping image(%s) because marker file not found" % image_id) continue try: if self._image_manager._delete_single_image( image_sweeper, image_dir, image_id): deleted_images.append(image_id) except Exception as ex: self._logger.warning("Failed to remove image: %s, %s" % (image_dir, ex)) continue waste_time(rest_interval_sec) # Now attempt GCing the image directory. try: self._image_manager._clean_gc_dir(image_sweeper.datastore_id) except Exception: # Swallow the exception the next clean call will clear it all. self._logger.exception("Failed to delete gc dir on datastore %s" % image_sweeper.datastore_id) return deleted_images
def _delete_unused_images(self, image_sweeper): deleted_images = list() target_images = image_sweeper.get_target_images() # Compute sweep rest interval rest_interval_sec = image_sweeper.get_image_sweep_rest_interval() for image_id in target_images: # On a directory change check if it still needs to run if image_sweeper.is_stopped(): return image_dir = image_directory_path(image_sweeper.datastore_id, image_id) # If there is not a marker file, skip it marker_pathname = os.path.join(image_dir, self._image_manager.IMAGE_MARKER_FILE_NAME) if not os.path.isfile(marker_pathname): self._logger.warn("skipping image(%s) because marker file not found" % image_id) continue try: if self._image_manager._delete_single_image(image_sweeper, image_dir, image_id): deleted_images.append(image_id) except Exception as ex: self._logger.warning("Failed to remove image: %s, %s" % (image_dir, ex)) continue waste_time(rest_interval_sec) # Now attempt GCing the image directory. try: self._image_manager._clean_gc_dir(image_sweeper.datastore_id) except Exception: # Swallow the exception the next clean call will clear it all. self._logger.exception("Failed to delete gc dir on datastore %s" % image_sweeper.datastore_id) return deleted_images