def _make_image_dir(self, datastore, image_id,
                        parent_folder_name=IMAGE_FOLDER_NAME):
        path = os.path.dirname(
            os_vmdk_path(
                datastore,
                image_id,
                parent_folder_name))

        if os.path.isdir(path):
            return

        # On shared volumes makedirs can fail with not found in rare corner
        # cases if two directory creates collide. Just retry in that case
        for attempt in range(1, self.NUM_MAKEDIRS_ATTEMPTS+1):
            try:
                mkdir_p(path)
            except OSError:
                self._logger.debug("Retrying (%u) while creating %s" %
                                   (attempt, path))
                if attempt == self.NUM_MAKEDIRS_ATTEMPTS:
                    raise
                else:
                    continue
            # Directory got created, stop the for loop
            break
    def _check_image_repair(self, image_id, datastore):
        vmdk_pathname = os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME)

        image_dirname = os.path.dirname(vmdk_pathname)
        try:
            # Check vmdk file
            if not os.path.exists(vmdk_pathname):
                self._logger.info("Vmdk path doesn't exists: %s" % vmdk_pathname)
                return False
        except Exception as ex:
            self._logger.exception("Exception validating %s, %s" % (image_dirname, ex))
            return False

        # Check timestamp file
        timestamp_pathname = os.path.join(image_dirname, self.IMAGE_TIMESTAMP_FILE_NAME)
        try:
            if os.path.exists(timestamp_pathname):
                self._logger.info("Timestamp file exists: %s" % timestamp_pathname)
                return True
        except Exception as ex:
            self._logger.exception("Exception validating %s, %s" % (timestamp_pathname, ex))

        # The timestamp file is not accessible,
        # try creating one, if successful try to
        # delete the renamed timestamp file if it
        # exists
        try:
            self._create_image_timestamp_file(image_dirname)
            self._delete_renamed_image_timestamp_file(image_dirname)
        except Exception as ex:
            self._logger.exception("Exception creating %s, %s" % (timestamp_pathname, ex))
            return False

        self._logger.info("Image repaired: %s" % image_dirname)
        return True
Esempio n. 3
0
    def _make_image_dir(self, datastore, image_id,
                        parent_folder_name=IMAGE_FOLDER_NAME_PREFIX):
        path = os.path.dirname(
            os_vmdk_path(
                datastore,
                image_id,
                parent_folder_name))

        if os.path.isdir(path):
            return

        # On shared volumes makedirs can fail with not found in rare corner
        # cases if two directory creates collide. Just retry in that case
        for attempt in range(1, self.NUM_MAKEDIRS_ATTEMPTS+1):
            try:
                mkdir_p(path)
            except OSError:
                self._logger.debug("Retrying (%u) while creating %s" %
                                   (attempt, path))
                if attempt == self.NUM_MAKEDIRS_ATTEMPTS:
                    raise
                else:
                    continue
            # Directory got created, stop the for loop
            break
    def touch_image_timestamp(self, ds_id, image_id):
        """
        :param ds_id:
        :param image_id:
        :return:
        """
        image_path = os.path.dirname(
            os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME))

        # Check the existence of the timestamp file
        tombstone_pathname = \
            os.path.join(image_path,
                         self.IMAGE_TOMBSTONE_FILE_NAME)
        try:
            tombstone = os.path.exists(tombstone_pathname)
        except Exception as ex:
            self._logger.exception(
                "Exception looking up %s, %s" % (tombstone_pathname, ex))

        if tombstone:
            raise InvalidImageState

        # Touch the timestamp file
        timestamp_pathname = \
            os.path.join(image_path,
                         self.IMAGE_TIMESTAMP_FILE_NAME)
        try:
            os.utime(timestamp_pathname, None)
        except Exception as ex:
            self._logger.exception(
                "Exception looking up %s, %s" % (timestamp_pathname, ex))
            raise ex
    def check_and_validate_image(self, image_id, ds_id):
        image_dir = os.path.dirname(
            os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME))

        try:
            if not os.path.exists(image_dir):
                return False
        except:
            self._logger.exception(
                "Error looking up %s" % image_dir)
            return False

        # Check the existence of the timestamp file
        timestamp_pathname = \
            os.path.join(image_dir,
                         self.IMAGE_TIMESTAMP_FILE_NAME)
        try:
            if os.path.exists(timestamp_pathname):
                return True
        except Exception as ex:
            self._logger.exception(
                "Exception looking up %s, %s" % (timestamp_pathname, ex))
            return False

        return False
 def check_image_dir(self, image_id, datastore):
     image_path = os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME)
     try:
         return os.path.exists(os.path.dirname(image_path))
     except:
         self._logger.error("Error looking up %s" % image_path, exc_info=True)
         return False
Esempio n. 7
0
    def touch_image_timestamp(self, ds_id, image_id):
        """
        :param ds_id:
        :param image_id:
        :return:
        """
        image_path = os.path.dirname(
            os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME))

        # Check the existence of the timestamp file
        tombstone_pathname = \
            os.path.join(image_path,
                         self.IMAGE_TOMBSTONE_FILE_NAME)
        try:
            tombstone = os.path.exists(tombstone_pathname)
        except Exception as ex:
            self._logger.exception("Exception looking up %s, %s" %
                                   (tombstone_pathname, ex))

        if tombstone:
            raise InvalidImageState

        # Touch the timestamp file
        timestamp_pathname = \
            os.path.join(image_path,
                         self.IMAGE_TIMESTAMP_FILE_NAME)
        try:
            os.utime(timestamp_pathname, None)
        except Exception as ex:
            self._logger.exception("Exception looking up %s, %s" %
                                   (timestamp_pathname, ex))
            raise ex
Esempio n. 8
0
    def check_and_validate_image(self, image_id, ds_id):
        image_dir = os.path.dirname(
            os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME_PREFIX))

        try:
            if not os.path.exists(image_dir):
                return False
        except:
            self._logger.exception(
                "Error looking up %s" % image_dir)
            return False

        # Check the existence of the timestamp file
        timestamp_pathname = \
            os.path.join(image_dir,
                         self.IMAGE_TIMESTAMP_FILE_NAME)
        try:
            if os.path.exists(timestamp_pathname):
                return True
        except Exception as ex:
            self._logger.exception(
                "Exception looking up %s, %s" % (timestamp_pathname, ex))
            return False

        return False
Esempio n. 9
0
 def check_image_dir(self, image_id, datastore):
     image_path = os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME)
     try:
         return os.path.exists(os.path.dirname(image_path))
     except:
         self._logger.error("Error looking up %s" % image_path,
                            exc_info=True)
         return False
    def check_image(self, image_id, datastore):
        image_dir = os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME)

        try:
            return os.path.exists(image_dir)
        except:
            self._logger.exception("Error looking up %s" % image_dir)
            return False
Esempio n. 11
0
    def check_image(self, image_id, datastore):
        image_dir = os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME)

        try:
            return os.path.exists(image_dir)
        except:
            self._logger.exception("Error looking up %s" % image_dir)
            return False
Esempio n. 12
0
    def _move_image(self, image_id, datastore, tmp_dir):
        """
        Atomic move of a tmp folder into the image datastore. Handles
        concurrent moves by locking a well know derivative of the image_id
        while doing the atomic move.
        The exclusive file lock ensures that only one move is successful.
        Has the following side effects:
            a - If the destination image already exists, it is assumed that
            someone else successfully copied the image over and the temp
            directory is deleted.
            b - If we fail to acquire the file lock after retrying 3 times,
            or the atomic move fails, the tmp image directory will be left
            behind and needs to be garbage collected later.

        image_id: String.The image id of the image being moved.
        datastore: String. The datastore id of the datastore.
        tmp_dir: String. The absolute path of the temp image directory.

        raises: OsError if the move fails
                AcquireLockFailure, InvalidFile if we fail to lock the
                destination image.
        """
        ds_type = self._get_datastore_type(datastore)
        image_path = os.path.dirname(
            os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME))
        parent_path = os.path.dirname(image_path)
        # Create the parent image directory if it doesn't exist.
        try:
            mkdir_p(parent_path)
        except OSError as e:
            if e.errno == errno.EEXIST and os.path.isdir(parent_path):
                # Parent directory exists nothing to do.
                pass
            else:
                raise
        try:
            with FileBackedLock(image_path, ds_type, retry=300,
                                wait_secs=0.01):  # wait lock for 3 seconds
                if self._check_image_repair(image_id, datastore):
                    raise DiskAlreadyExistException("Image already exists")

                shutil.move(tmp_dir, image_path)
        except (AcquireLockFailure, InvalidFile):
            self._logger.info("Unable to lock %s for atomic move" % image_id)
            raise
        except DiskAlreadyExistException:
            self._logger.info("Image %s already copied" % image_id)
            rm_rf(tmp_dir)
            raise
    def _move_image(self, image_id, datastore, tmp_dir):
        """
        Atomic move of a tmp folder into the image datastore. Handles
        concurrent moves by locking a well know derivative of the image_id
        while doing the atomic move.
        The exclusive file lock ensures that only one move is successful.
        Has the following side effects:
            a - If the destination image already exists, it is assumed that
            someone else successfully copied the image over and the temp
            directory is deleted.
            b - If we fail to acquire the file lock after retrying 3 times,
            or the atomic move fails, the tmp image directory will be left
            behind and needs to be garbage collected later.

        image_id: String.The image id of the image being moved.
        datastore: String. The datastore id of the datastore.
        tmp_dir: String. The absolute path of the temp image directory.

        raises: OsError if the move fails
                AcquireLockFailure, InvalidFile if we fail to lock the
                destination image.
        """
        ds_type = self._get_datastore_type(datastore)
        image_path = os.path.dirname(os_vmdk_path(datastore, image_id,
                                                  IMAGE_FOLDER_NAME))
        parent_path = os.path.dirname(image_path)
        # Create the parent image directory if it doesn't exist.
        try:
            mkdir_p(parent_path)
        except OSError as e:
            if e.errno == errno.EEXIST and os.path.isdir(parent_path):
                # Parent directory exists nothing to do.
                pass
            else:
                raise
        try:
            with FileBackedLock(image_path, ds_type, retry=300,
                                wait_secs=0.01):  # wait lock for 3 seconds
                if self._check_image_repair(image_id, datastore):
                    raise DiskAlreadyExistException("Image already exists")

                shutil.move(tmp_dir, image_path)
        except (AcquireLockFailure, InvalidFile):
            self._logger.info("Unable to lock %s for atomic move" % image_id)
            raise
        except DiskAlreadyExistException:
            self._logger.info("Image %s already copied" % image_id)
            rm_rf(tmp_dir)
            raise
Esempio n. 14
0
def vmdk_mkdir(datastore, disk_id, logger):
    path = os.path.dirname(os_vmdk_path(datastore, disk_id))
    # On shared volumes makedirs can fail with not found in rare corner
    # cases if two directory creates collide. Just retry in that case
    for attempt in range(1, 10):
        try:
            os.makedirs(path)
        except OSError:
            logger.debug("Retrying (%u) while creating %s" % (attempt, path))
            if attempt == 10:
                raise
            else:
                continue
        # Directory got created, stop the for loop
        break
Esempio n. 15
0
def vmdk_mkdir(datastore, disk_id, logger):
    path = os.path.dirname(os_vmdk_path(datastore, disk_id))
    # On shared volumes makedirs can fail with not found in rare corner
    # cases if two directory creates collide. Just retry in that case
    for attempt in range(1, 10):
        try:
            os.makedirs(path)
        except OSError:
            logger.debug("Retrying (%u) while creating %s" %
                         (attempt, path))
            if attempt == 10:
                raise
            else:
                continue
        # Directory got created, stop the for loop
        break
Esempio n. 16
0
    def touch_image_timestamp(self, ds_id, image_id):
        """
        :param ds_id:
        :param image_id:
        :return:
        """
        image_path = os.path.dirname(
            os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME_PREFIX))

        # Touch the timestamp file
        timestamp_pathname = os.path.join(image_path, self.IMAGE_TIMESTAMP_FILE_NAME)
        try:
            os.utime(timestamp_pathname, None)
        except Exception as ex:
            self._logger.exception(
                "Exception looking up %s, %s" % (timestamp_pathname, ex))
            raise ex
    def create_image_tombstone(self, ds_id, image_id):
        """
        :param ds_id:
        :param image_id:
        :return:
        """
        image_path = os.path.dirname(os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME))

        # Create tombstone file for the image
        tombstone_pathname = os.path.join(image_path, self.IMAGE_TOMBSTONE_FILE_NAME)
        try:
            open(tombstone_pathname, "w").close()
        except Exception as ex:
            self._logger.exception("Exception creating %s, %s" % (tombstone_pathname, ex))
            raise ex

        self._logger.info("Image: %s tombstoned" % tombstone_pathname)
Esempio n. 18
0
    def touch_image_timestamp(self, ds_id, image_id):
        """
        :param ds_id:
        :param image_id:
        :return:
        """
        image_path = os.path.dirname(
            os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME_PREFIX))

        # Touch the timestamp file
        timestamp_pathname = os.path.join(image_path, self.IMAGE_TIMESTAMP_FILE_NAME)
        try:
            os.utime(timestamp_pathname, None)
        except Exception as ex:
            self._logger.exception(
                "Exception looking up %s, %s" % (timestamp_pathname, ex))
            raise ex
Esempio n. 19
0
    def _check_image_repair(self, image_id, datastore):
        vmdk_pathname = os_vmdk_path(datastore,
                                     image_id,
                                     IMAGE_FOLDER_NAME_PREFIX)

        image_dirname = os.path.dirname(vmdk_pathname)
        try:
            # Check vmdk file
            if not os.path.exists(vmdk_pathname):
                self._logger.info("Vmdk path doesn't exists: %s" %
                                  vmdk_pathname)
                return False
        except Exception as ex:
            self._logger.exception(
                "Exception validating %s, %s" % (image_dirname, ex))
            return False

        # Check timestamp file
        timestamp_pathname = \
            os.path.join(image_dirname,
                         self.IMAGE_TIMESTAMP_FILE_NAME)
        try:
            if os.path.exists(timestamp_pathname):
                self._logger.info("Timestamp file exists: %s" %
                                  timestamp_pathname)
                return True
        except Exception as ex:
            self._logger.exception(
                "Exception validating %s, %s" % (timestamp_pathname, ex))

        # The timestamp file is not accessible,
        # try creating one, if successful try to
        # delete the renamed timestamp file if it
        # exists
        try:
            self._create_image_timestamp_file(image_dirname)
            self._delete_renamed_image_timestamp_file(image_dirname)
        except Exception as ex:
            self._logger.exception(
                "Exception creating %s, %s" % (timestamp_pathname, ex))
            return False

        self._logger.info("Image repaired: %s" %
                          image_dirname)
        return True
Esempio n. 20
0
    def create_image_tombstone(self, ds_id, image_id):
        """
        :param ds_id:
        :param image_id:
        :return:
        """
        image_path = os.path.dirname(
            os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME))

        # Create tombstone file for the image
        tombstone_pathname = \
            os.path.join(image_path,
                         self.IMAGE_TOMBSTONE_FILE_NAME)
        try:
            open(tombstone_pathname, 'w').close()
        except Exception as ex:
            self._logger.exception("Exception creating %s, %s" %
                                   (tombstone_pathname, ex))
            raise ex

        self._logger.info("Image: %s tombstoned" % tombstone_pathname)
 def _gc_image_dir(self, datastore_id, image_id):
     """
     Moves the current image directory into the GC image folder and
     relies on the later GC call for cleanup.
     Exception is thrown if the move fails.
     Assumes the ref files contained in the image director are locked, so
     there can only be one move happening at any point in time.
     """
     src_path = os.path.dirname(os_vmdk_path(datastore_id, image_id, IMAGE_FOLDER_NAME))
     # Verify locking held.
     assert os.path.exists(src_path)
     # Generate a random suffix, this is to address the following.
     # a - Image disk say i1 was moved to the gc_dir
     # b - Cleanup of i1 failed for whatever reason.
     # c - New copy of i1 was uploaded to the datastore.
     # d - The new copy is now being deleted.
     # The location of the move in d needs to be different from the
     # location of the move from b, hence use a random uuid.
     rnd_uuid = str(uuid.uuid4())
     gc_dir = os_datastore_path(datastore_id, GC_IMAGE_FOLDER)
     dst_dir = os.path.join(gc_dir, rnd_uuid)
     os.makedirs(dst_dir)
     shutil.move(src_path, dst_dir)
Esempio n. 22
0
 def _gc_image_dir(self, datastore_id, image_id):
     """
     Moves the current image directory into the GC image folder and
     relies on the later GC call for cleanup.
     Exception is thrown if the move fails.
     Assumes the ref files contained in the image director are locked, so
     there can only be one move happening at any point in time.
     """
     src_path = os.path.dirname(
         os_vmdk_path(datastore_id, image_id, IMAGE_FOLDER_NAME))
     # Verify locking held.
     assert (os.path.exists(src_path))
     # Generate a random suffix, this is to address the following.
     # a - Image disk say i1 was moved to the gc_dir
     # b - Cleanup of i1 failed for whatever reason.
     # c - New copy of i1 was uploaded to the datastore.
     # d - The new copy is now being deleted.
     # The location of the move in d needs to be different from the
     # location of the move from b, hence use a random uuid.
     rnd_uuid = str(uuid.uuid4())
     gc_dir = os_datastore_path(datastore_id, GC_IMAGE_FOLDER)
     dst_dir = os.path.join(gc_dir, rnd_uuid)
     os.makedirs(dst_dir)
     shutil.move(src_path, dst_dir)
Esempio n. 23
0
 def get_datastore(self, disk_id):
     for datastore in self._ds_manager.get_datastore_ids():
         disk = os_vmdk_path(datastore, disk_id)
         if os.path.isfile(disk):
             return datastore
     return None
 def _create_image_timestamp_file_from_ids(self, ds_id, image_id):
     image_path = os.path.dirname(
         os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME))
     self._create_image_timestamp_file(image_path)
Esempio n. 25
0
 def _rm_image_dir(self,
                   datastore,
                   id,
                   parent_folder_name=IMAGE_FOLDER_NAME):
     path = os.path.dirname(os_vmdk_path(datastore, id, parent_folder_name))
     shutil.rmtree(path)
 def get_image_path(self, datastore_id, image_id):
     return os_vmdk_path(datastore_id, image_id, IMAGE_FOLDER_NAME)
Esempio n. 27
0
 def get_image_path(self, datastore_id, image_id):
     return os_vmdk_path(datastore_id, image_id, IMAGE_FOLDER_NAME)
Esempio n. 28
0
 def _vmdk_rmdir(self, datastore, disk_id):
     path = os.path.dirname(os_vmdk_path(datastore, disk_id))
     shutil.rmtree(path)
 def _rm_image_dir(self, datastore, id,
                   parent_folder_name=IMAGE_FOLDER_NAME):
     path = os.path.dirname(os_vmdk_path(datastore, id, parent_folder_name))
     shutil.rmtree(path)
Esempio n. 30
0
 def _create_image_timestamp_file_from_ids(self, ds_id, image_id):
     image_path = os.path.dirname(
         os_vmdk_path(ds_id, image_id, IMAGE_FOLDER_NAME))
     self._create_image_timestamp_file(image_path)
 def patched_os_vmdk_path(self, datastore, disk_id, folder):
     folder = self.dir0
     ret = os_vmdk_path(datastore, disk_id, folder)
     return ret
 def patched_os_vmdk_path(self, datastore, disk_id, folder):
     folder = self.dir0
     ret = os_vmdk_path(datastore, disk_id, folder)
     return ret
Esempio n. 33
0
 def _vmdk_rmdir(self, datastore, disk_id):
     path = os.path.dirname(os_vmdk_path(datastore, disk_id))
     shutil.rmtree(path)
Esempio n. 34
0
 def _vmdk_rmdir(self, datastore, disk_id):
     path = os.path.dirname(os_vmdk_path(datastore, disk_id))
     self._vim_client.delete_file(path)
Esempio n. 35
0
 def _vmdk_mkdir(self, datastore, disk_id):
     path = os.path.dirname(os_vmdk_path(datastore, disk_id))
     self._vim_client.make_directory(path)
Esempio n. 36
0
 def _vmdk_mkdir(self, datastore, disk_id):
     path = os.path.dirname(os_vmdk_path(datastore, disk_id))
     self._vim_client.make_directory(path)
Esempio n. 37
0
 def get_datastore(self, disk_id):
     for datastore in self._ds_manager.get_datastore_ids():
         disk = os_vmdk_path(datastore, disk_id)
         if os.path.isfile(disk):
             return datastore
     return None
Esempio n. 38
0
 def _vmdk_rmdir(self, datastore, disk_id):
     path = os.path.dirname(os_vmdk_path(datastore, disk_id))
     self._vim_client.delete_file(path)