def test_convert_bytes_to_gb(self): # A round 1 GB test = util.convert_bytes_to_gb(1024 * 1024 * 1024) self.assertEqual(1.0, test) # A single MB test = util.convert_bytes_to_gb(1024 * 1024.0) self.assertEqual(0.0009765625, test) # A single byte - should be the low Value self.assertEqual(.0001, util.convert_bytes_to_gb(1)) # Try changing the low value self.assertEqual(.0005, util.convert_bytes_to_gb(1, .0005)) # Round up self.assertEqual(1.15, util.convert_bytes_to_gb(1224067890, dp=2))
def upload_new_vdisk(adapter, v_uuid, vol_grp_uuid, d_stream, d_name, f_size, d_size=None, sha_chksum=None): """Creates a new Virtual Disk and uploads a data stream to it. :param adapter: The adapter to talk over the API. :param v_uuid: The Virtual I/O Server UUID that will host the disk. :param vol_grp_uuid: The volume group that will host the Virtual Disk's UUID. :param d_stream: The data stream (either a file handle or stream) to upload. Must have the 'read' method that returns a chunk of bytes. :param d_name: The name that should be given to the disk on the Virtual I/O Server that will contain the file. :param f_size: The size (in bytes) of the stream to be uploaded. :param d_size: (OPTIONAL) The desired size of the new VDisk in bytes. If omitted or smaller than f_size, it will be set to match f_size. :param sha_chksum: (OPTIONAL) The SHA256 checksum for the file. Useful for integrity checks. :return: The first return value is the virtual disk that the file is uploaded into. :return: Normally the second return value will be None, indicating that the disk and image were uploaded without issue. If for some reason the File metadata for the VIOS was not cleaned up, the return value is the File EntryWrapper. This is simply a metadata marker to be later used to retry the cleanup. """ # Create the new virtual disk. The size here is in GB. We can use decimal # precision on the create call. What the VIOS will then do is determine # the appropriate segment size (pp) and will provide a virtual disk that # is 'at least' that big. Depends on the segment size set up on the # volume group how much over it could go. # # See note below...temporary workaround needed. if d_size is None or d_size < f_size: d_size = f_size gb_size = util.convert_bytes_to_gb(d_size) # TODO(IBM) Temporary - need to round up to the highest GB. This should # be done by the platform in the future. gb_size = math.ceil(gb_size) n_vdisk = crt_vdisk(adapter, v_uuid, vol_grp_uuid, d_name, gb_size) # The file type. If local API server, then we can use the coordinated # file path. Otherwise standard upload. file_type = (vf.FileType.DISK_IMAGE_COORDINATED if adapter.traits.local_api else vf.FileType.DISK_IMAGE) # Next, create the file, but specify the appropriate disk udid from the # Virtual Disk vio_file = _create_file( adapter, d_name, file_type, v_uuid, f_size=f_size, tdev_udid=n_vdisk.udid, sha_chksum=sha_chksum) maybe_file = _upload_stream(vio_file, d_stream) return n_vdisk, maybe_file
def upload_new_lu(v_uuid, ssp, d_stream, lu_name, f_size, d_size=None, sha_chksum=None): """Creates a new SSP Logical Unit and uploads a data stream to it. :param v_uuid: The UUID of the Virtual I/O Server through which to perform the upload. (Note that the new LU will be visible from any VIOS in the Shared Storage Pool's Cluster.) :param ssp: SSP EntryWrapper representing the Shared Storage Pool on which to create the new Logical Unit. :param d_stream: The data stream (either a file handle or stream) to upload. Must have the 'read' method that returns a chunk of bytes. :param lu_name: The name that should be given to the new LU. :param f_size: The size (in bytes) of the stream to be uploaded. :param d_size: (OPTIONAL) The size of the LU (in bytes). Not required if it should match the file. Must be at least as large as the file. :param sha_chksum: (OPTIONAL) The SHA256 checksum for the file. Useful for integrity checks. :return: The first return value is an LU EntryWrapper corresponding to the Logical Unit into which the file was uploaded. :return: Normally the second return value will be None, indicating that the LU was created and the image was uploaded without issue. If for some reason the File metadata for the VIOS was not cleaned up, the return value is the LU EntryWrapper. This is simply a marker to be later used to retry the cleanup. """ # Create the new Logical Unit. The LU size needs to be in decimal GB. if d_size is None or d_size < f_size: d_size = f_size gb_size = util.convert_bytes_to_gb(d_size, dp=2) ssp, new_lu = crt_lu(ssp, lu_name, gb_size, typ=stor.LUType.IMAGE) # The file type. If local API server, then we can use the coordinated # file path. Otherwise standard upload. file_type = (vf.FileType.DISK_IMAGE_COORDINATED if ssp.adapter.traits.local_api else vf.FileType.DISK_IMAGE) # Create the file, specifying the UDID from the new Logical Unit. # The File name matches the LU name. vio_file = _create_file(ssp.adapter, lu_name, file_type, v_uuid, f_size=f_size, tdev_udid=new_lu.udid, sha_chksum=sha_chksum) maybe_file = _upload_stream(vio_file, d_stream) return new_lu, maybe_file
def upload_new_lu(v_uuid, ssp, d_stream, lu_name, f_size, d_size=None, sha_chksum=None): """Creates a new SSP Logical Unit and uploads a data stream to it. :param v_uuid: The UUID of the Virtual I/O Server through which to perform the upload. (Note that the new LU will be visible from any VIOS in the Shared Storage Pool's Cluster.) :param ssp: SSP EntryWrapper representing the Shared Storage Pool on which to create the new Logical Unit. :param d_stream: The data stream (either a file handle or stream) to upload. Must have the 'read' method that returns a chunk of bytes. :param lu_name: The name that should be given to the new LU. :param f_size: The size (in bytes) of the stream to be uploaded. :param d_size: (OPTIONAL) The size of the LU (in bytes). Not required if it should match the file. Must be at least as large as the file. :param sha_chksum: (OPTIONAL) The SHA256 checksum for the file. Useful for integrity checks. :return: The first return value is an LU EntryWrapper corresponding to the Logical Unit into which the file was uploaded. :return: Normally the second return value will be None, indicating that the LU was created and the image was uploaded without issue. If for some reason the File metadata for the VIOS was not cleaned up, the return value is the LU EntryWrapper. This is simply a marker to be later used to retry the cleanup. """ # Create the new Logical Unit. The LU size needs to be in decimal GB. if d_size is None or d_size < f_size: d_size = f_size gb_size = util.convert_bytes_to_gb(d_size, dp=2) ssp, new_lu = crt_lu(ssp, lu_name, gb_size, typ=stor.LUType.IMAGE) # The file type. If local API server, then we can use the coordinated # file path. Otherwise standard upload. file_type = (vf.FileType.DISK_IMAGE_COORDINATED if ssp.adapter.traits.local_api else vf.FileType.DISK_IMAGE) # Create the file, specifying the UDID from the new Logical Unit. # The File name matches the LU name. vio_file = _create_file( ssp.adapter, lu_name, file_type, v_uuid, f_size=f_size, tdev_udid=new_lu.udid, sha_chksum=sha_chksum) maybe_file = _upload_stream(vio_file, d_stream) return new_lu, maybe_file
def get_or_upload_image_lu(tier, luname, vios_uuid, io_handle, b_size, upload_type=tsk_stg.UploadType.IO_STREAM_BUILDER): """Ensures our SSP has an LU containing the specified image. If an LU of type IMAGE with the specified luname already exists in our SSP, return it. Otherwise, create it, prime it with the image contents provided via stream_func, and return it. This method assumes that consumers employ a naming convention such that an LU with a given name represents the same data (size and content) no matter where/when it's created/uploaded - for example, by including the image's MD5 checksum in the name. This method is designed to coordinate the upload of a particular image LU across multiple hosts which use the same SSP, but otherwise can not communicate with each other. :param tier: Tier EntryWrapper of the Shared Storage Pool Tier on which the image LU is to be hosted. :param luname: The name of the image LU. Note that the name may be shortened to satisfy length restrictions. :param vios_uuid: The UUID of the Virtual I/O Server through which the upload should be performed, if necessary. :param io_handle: The I/O handle (as defined by the upload_type). This is only used if the image_lu needs to be uploaded. :param b_size: Integer size, in bytes, of the image provided by stream_func's return value. :param upload_type: (Optional, Default: IO_STREAM_BUILDER) Defines the way in which the LU should be uploaded. Refer to the UploadType enumeration for valid upload mechanisms. It defaults to IO_STREAM_BUILDER for legacy reasons. :return: LUEnt EntryWrapper representing the image LU. """ # Marker (upload-in-progress) LU name prefixed with 'partxxxxxxxx' prefix = 'part%s' % uuid.uuid4().hex[:8] # Ensure the marker LU name won't be too long luname = u.sanitize_file_name_for_api(luname, max_len=c.MaxLen.FILENAME_DEFAULT - len(prefix)) mkr_luname = prefix + luname first = True while True: # (Re)fetch the list of image LUs whose name *contains* luname. lus = _find_lus(tier, luname) # Does the LU already exist in its final, uploaded form? If so, then # only that LU will exist, with an exact name match. if len(lus) == 1 and lus[0].name == luname: LOG.info(_('Using already-uploaded image LU %s.'), luname) return lus[0] # Is there an upload in progress? if _upload_in_progress(lus, luname, first): first = False _sleep_for_upload() continue # No upload in progress (at least as of when we grabbed the feed). LOG.info(_('Creating marker LU %s'), mkr_luname) tier, mkrlu = tsk_stg.crt_lu(tier, mkr_luname, MKRSZ, typ=IMGTYP) # We must remove the marker LU if # a) anything fails beyond this point; or # b) we successfully upload the image LU. try: # If another process (possibly on another host) created a marker LU # at the same time, there could be multiple marker LUs out there. # We all use _upload_conflict to decide which one of us gets to do # the upload. if _upload_conflict(tier, luname, mkr_luname): _sleep_for_upload() continue # Okay, we won. Do the actual upload. LOG.info(_('Uploading to image LU %(lu)s (marker %(mkr)s).'), { 'lu': luname, 'mkr': mkr_luname }) # Create the new Logical Unit. The LU size needs to be decimal GB. tier, new_lu = tsk_stg.crt_lu(tier, luname, u.convert_bytes_to_gb(b_size, dp=2), typ=IMGTYP) try: tsk_stg.upload_lu(vios_uuid, new_lu, io_handle, b_size, upload_type=upload_type) except Exception as exc: LOG.exception(exc) # We need to remove the LU so it doesn't block others # attempting to use the same one. LOG.exception(_('Removing failed LU %s.'), luname) new_lu.delete() raise return new_lu finally: # Signal completion, or clean up, by removing the marker LU. mkrlu.delete()
def upload_new_vdisk(adapter, v_uuid, vol_grp_uuid, d_stream, d_name, f_size, d_size=None, sha_chksum=None): """Creates a new Virtual Disk and uploads a data stream to it. :param adapter: The adapter to talk over the API. :param v_uuid: The Virtual I/O Server UUID that will host the disk. :param vol_grp_uuid: The volume group that will host the Virtual Disk's UUID. :param d_stream: The data stream (either a file handle or stream) to upload. Must have the 'read' method that returns a chunk of bytes. :param d_name: The name that should be given to the disk on the Virtual I/O Server that will contain the file. :param f_size: The size (in bytes) of the stream to be uploaded. :param d_size: (OPTIONAL) The desired size of the new VDisk in bytes. If omitted or smaller than f_size, it will be set to match f_size. :param sha_chksum: (OPTIONAL) The SHA256 checksum for the file. Useful for integrity checks. :return: The first return value is the virtual disk that the file is uploaded into. :return: Normally the second return value will be None, indicating that the disk and image were uploaded without issue. If for some reason the File metadata for the VIOS was not cleaned up, the return value is the File EntryWrapper. This is simply a metadata marker to be later used to retry the cleanup. """ # Create the new virtual disk. The size here is in GB. We can use decimal # precision on the create call. What the VIOS will then do is determine # the appropriate segment size (pp) and will provide a virtual disk that # is 'at least' that big. Depends on the segment size set up on the # volume group how much over it could go. # # See note below...temporary workaround needed. if d_size is None or d_size < f_size: d_size = f_size gb_size = util.convert_bytes_to_gb(d_size) # TODO(IBM) Temporary - need to round up to the highest GB. This should # be done by the platform in the future. gb_size = math.ceil(gb_size) n_vdisk = crt_vdisk(adapter, v_uuid, vol_grp_uuid, d_name, gb_size) # The file type. If local API server, then we can use the coordinated # file path. Otherwise standard upload. file_type = (vf.FileType.DISK_IMAGE_COORDINATED if adapter.traits.local_api else vf.FileType.DISK_IMAGE) # Next, create the file, but specify the appropriate disk udid from the # Virtual Disk vio_file = _create_file(adapter, d_name, file_type, v_uuid, f_size=f_size, tdev_udid=n_vdisk.udid, sha_chksum=sha_chksum) maybe_file = _upload_stream(vio_file, d_stream) return n_vdisk, maybe_file