def test_convert_bytes_to_gb(self):
# A round 1 GB
test = util.convert_bytes_to_gb(1024 * 1024 * 1024)
self.assertEqual(1.0, test)
# A single MB
test = util.convert_bytes_to_gb(1024 * 1024.0)
self.assertEqual(0.0009765625, test)
# A single byte - should be the low Value
self.assertEqual(.0001, util.convert_bytes_to_gb(1))
# Try changing the low value
self.assertEqual(.0005, util.convert_bytes_to_gb(1, .0005))
# Round up
self.assertEqual(1.15, util.convert_bytes_to_gb(1224067890, dp=2))
def test_convert_bytes_to_gb(self):
# A round 1 GB
test = util.convert_bytes_to_gb(1024 * 1024 * 1024)
self.assertEqual(1.0, test)
# A single MB
test = util.convert_bytes_to_gb(1024 * 1024.0)
self.assertEqual(0.0009765625, test)
# A single byte - should be the low Value
self.assertEqual(.0001, util.convert_bytes_to_gb(1))
# Try changing the low value
self.assertEqual(.0005, util.convert_bytes_to_gb(1, .0005))
# Round up
self.assertEqual(1.15, util.convert_bytes_to_gb(1224067890, dp=2))
def upload_new_vdisk(adapter, v_uuid, vol_grp_uuid, d_stream,
d_name, f_size, d_size=None, sha_chksum=None):
"""Creates a new Virtual Disk and uploads a data stream to it.
:param adapter: The adapter to talk over the API.
:param v_uuid: The Virtual I/O Server UUID that will host the disk.
:param vol_grp_uuid: The volume group that will host the Virtual Disk's
UUID.
:param d_stream: The data stream (either a file handle or stream) to
upload. Must have the 'read' method that returns a chunk
of bytes.
:param d_name: The name that should be given to the disk on the Virtual
I/O Server that will contain the file.
:param f_size: The size (in bytes) of the stream to be uploaded.
:param d_size: (OPTIONAL) The desired size of the new VDisk in bytes. If
omitted or smaller than f_size, it will be set to match
f_size.
:param sha_chksum: (OPTIONAL) The SHA256 checksum for the file. Useful for
integrity checks.
:return: The first return value is the virtual disk that the file is
uploaded into.
:return: Normally the second return value will be None, indicating that the
disk and image were uploaded without issue. If for some reason
the File metadata for the VIOS was not cleaned up, the return
value is the File EntryWrapper. This is simply a metadata marker
to be later used to retry the cleanup.
"""
# Create the new virtual disk. The size here is in GB. We can use decimal
# precision on the create call. What the VIOS will then do is determine
# the appropriate segment size (pp) and will provide a virtual disk that
# is 'at least' that big. Depends on the segment size set up on the
# volume group how much over it could go.
#
# See note below...temporary workaround needed.
if d_size is None or d_size < f_size:
d_size = f_size
gb_size = util.convert_bytes_to_gb(d_size)
# TODO(IBM) Temporary - need to round up to the highest GB. This should
# be done by the platform in the future.
gb_size = math.ceil(gb_size)
n_vdisk = crt_vdisk(adapter, v_uuid, vol_grp_uuid, d_name, gb_size)
# The file type. If local API server, then we can use the coordinated
# file path. Otherwise standard upload.
file_type = (vf.FileType.DISK_IMAGE_COORDINATED if adapter.traits.local_api
else vf.FileType.DISK_IMAGE)
# Next, create the file, but specify the appropriate disk udid from the
# Virtual Disk
vio_file = _create_file(
adapter, d_name, file_type, v_uuid, f_size=f_size,
tdev_udid=n_vdisk.udid, sha_chksum=sha_chksum)
maybe_file = _upload_stream(vio_file, d_stream)
return n_vdisk, maybe_file
def upload_new_lu(v_uuid,
ssp,
d_stream,
lu_name,
f_size,
d_size=None,
sha_chksum=None):
"""Creates a new SSP Logical Unit and uploads a data stream to it.
:param v_uuid: The UUID of the Virtual I/O Server through which to perform
the upload. (Note that the new LU will be visible from any
VIOS in the Shared Storage Pool's Cluster.)
:param ssp: SSP EntryWrapper representing the Shared Storage Pool on which
to create the new Logical Unit.
:param d_stream: The data stream (either a file handle or stream) to
upload. Must have the 'read' method that returns a chunk
of bytes.
:param lu_name: The name that should be given to the new LU.
:param f_size: The size (in bytes) of the stream to be uploaded.
:param d_size: (OPTIONAL) The size of the LU (in bytes). Not required if
it should match the file. Must be at least as large as the
file.
:param sha_chksum: (OPTIONAL) The SHA256 checksum for the file. Useful for
integrity checks.
:return: The first return value is an LU EntryWrapper corresponding to the
Logical Unit into which the file was uploaded.
:return: Normally the second return value will be None, indicating that the
LU was created and the image was uploaded without issue. If for
some reason the File metadata for the VIOS was not cleaned up, the
return value is the LU EntryWrapper. This is simply a marker to
be later used to retry the cleanup.
"""
# Create the new Logical Unit. The LU size needs to be in decimal GB.
if d_size is None or d_size < f_size:
d_size = f_size
gb_size = util.convert_bytes_to_gb(d_size, dp=2)
ssp, new_lu = crt_lu(ssp, lu_name, gb_size, typ=stor.LUType.IMAGE)
# The file type. If local API server, then we can use the coordinated
# file path. Otherwise standard upload.
file_type = (vf.FileType.DISK_IMAGE_COORDINATED
if ssp.adapter.traits.local_api else vf.FileType.DISK_IMAGE)
# Create the file, specifying the UDID from the new Logical Unit.
# The File name matches the LU name.
vio_file = _create_file(ssp.adapter,
lu_name,
file_type,
v_uuid,
f_size=f_size,
tdev_udid=new_lu.udid,
sha_chksum=sha_chksum)
maybe_file = _upload_stream(vio_file, d_stream)
return new_lu, maybe_file
def upload_new_lu(v_uuid, ssp, d_stream, lu_name, f_size, d_size=None,
sha_chksum=None):
"""Creates a new SSP Logical Unit and uploads a data stream to it.
:param v_uuid: The UUID of the Virtual I/O Server through which to perform
the upload. (Note that the new LU will be visible from any
VIOS in the Shared Storage Pool's Cluster.)
:param ssp: SSP EntryWrapper representing the Shared Storage Pool on which
to create the new Logical Unit.
:param d_stream: The data stream (either a file handle or stream) to
upload. Must have the 'read' method that returns a chunk
of bytes.
:param lu_name: The name that should be given to the new LU.
:param f_size: The size (in bytes) of the stream to be uploaded.
:param d_size: (OPTIONAL) The size of the LU (in bytes). Not required if
it should match the file. Must be at least as large as the
file.
:param sha_chksum: (OPTIONAL) The SHA256 checksum for the file. Useful for
integrity checks.
:return: The first return value is an LU EntryWrapper corresponding to the
Logical Unit into which the file was uploaded.
:return: Normally the second return value will be None, indicating that the
LU was created and the image was uploaded without issue. If for
some reason the File metadata for the VIOS was not cleaned up, the
return value is the LU EntryWrapper. This is simply a marker to
be later used to retry the cleanup.
"""
# Create the new Logical Unit. The LU size needs to be in decimal GB.
if d_size is None or d_size < f_size:
d_size = f_size
gb_size = util.convert_bytes_to_gb(d_size, dp=2)
ssp, new_lu = crt_lu(ssp, lu_name, gb_size, typ=stor.LUType.IMAGE)
# The file type. If local API server, then we can use the coordinated
# file path. Otherwise standard upload.
file_type = (vf.FileType.DISK_IMAGE_COORDINATED
if ssp.adapter.traits.local_api
else vf.FileType.DISK_IMAGE)
# Create the file, specifying the UDID from the new Logical Unit.
# The File name matches the LU name.
vio_file = _create_file(
ssp.adapter, lu_name, file_type, v_uuid, f_size=f_size,
tdev_udid=new_lu.udid, sha_chksum=sha_chksum)
maybe_file = _upload_stream(vio_file, d_stream)
return new_lu, maybe_file
def get_or_upload_image_lu(tier,
luname,
vios_uuid,
io_handle,
b_size,
upload_type=tsk_stg.UploadType.IO_STREAM_BUILDER):
"""Ensures our SSP has an LU containing the specified image.
If an LU of type IMAGE with the specified luname already exists in our SSP,
return it. Otherwise, create it, prime it with the image contents provided
via stream_func, and return it.
This method assumes that consumers employ a naming convention such that an
LU with a given name represents the same data (size and content) no matter
where/when it's created/uploaded - for example, by including the image's
MD5 checksum in the name.
This method is designed to coordinate the upload of a particular image LU
across multiple hosts which use the same SSP, but otherwise can not
communicate with each other.
:param tier: Tier EntryWrapper of the Shared Storage Pool Tier on which the
image LU is to be hosted.
:param luname: The name of the image LU. Note that the name may be
shortened to satisfy length restrictions.
:param vios_uuid: The UUID of the Virtual I/O Server through which the
upload should be performed, if necessary.
:param io_handle: The I/O handle (as defined by the upload_type). This is
only used if the image_lu needs to be uploaded.
:param b_size: Integer size, in bytes, of the image provided by
stream_func's return value.
:param upload_type: (Optional, Default: IO_STREAM_BUILDER) Defines the way
in which the LU should be uploaded. Refer to the
UploadType enumeration for valid upload mechanisms.
It defaults to IO_STREAM_BUILDER for legacy reasons.
:return: LUEnt EntryWrapper representing the image LU.
"""
# Marker (upload-in-progress) LU name prefixed with 'partxxxxxxxx'
prefix = 'part%s' % uuid.uuid4().hex[:8]
# Ensure the marker LU name won't be too long
luname = u.sanitize_file_name_for_api(luname,
max_len=c.MaxLen.FILENAME_DEFAULT -
len(prefix))
mkr_luname = prefix + luname
first = True
while True:
# (Re)fetch the list of image LUs whose name *contains* luname.
lus = _find_lus(tier, luname)
# Does the LU already exist in its final, uploaded form? If so, then
# only that LU will exist, with an exact name match.
if len(lus) == 1 and lus[0].name == luname:
LOG.info(_('Using already-uploaded image LU %s.'), luname)
return lus[0]
# Is there an upload in progress?
if _upload_in_progress(lus, luname, first):
first = False
_sleep_for_upload()
continue
# No upload in progress (at least as of when we grabbed the feed).
LOG.info(_('Creating marker LU %s'), mkr_luname)
tier, mkrlu = tsk_stg.crt_lu(tier, mkr_luname, MKRSZ, typ=IMGTYP)
# We must remove the marker LU if
# a) anything fails beyond this point; or
# b) we successfully upload the image LU.
try:
# If another process (possibly on another host) created a marker LU
# at the same time, there could be multiple marker LUs out there.
# We all use _upload_conflict to decide which one of us gets to do
# the upload.
if _upload_conflict(tier, luname, mkr_luname):
_sleep_for_upload()
continue
# Okay, we won. Do the actual upload.
LOG.info(_('Uploading to image LU %(lu)s (marker %(mkr)s).'), {
'lu': luname,
'mkr': mkr_luname
})
# Create the new Logical Unit. The LU size needs to be decimal GB.
tier, new_lu = tsk_stg.crt_lu(tier,
luname,
u.convert_bytes_to_gb(b_size, dp=2),
typ=IMGTYP)
try:
tsk_stg.upload_lu(vios_uuid,
new_lu,
io_handle,
b_size,
upload_type=upload_type)
except Exception as exc:
LOG.exception(exc)
# We need to remove the LU so it doesn't block others
# attempting to use the same one.
LOG.exception(_('Removing failed LU %s.'), luname)
new_lu.delete()
raise
return new_lu
finally:
# Signal completion, or clean up, by removing the marker LU.
mkrlu.delete()
def upload_new_vdisk(adapter,
v_uuid,
vol_grp_uuid,
d_stream,
d_name,
f_size,
d_size=None,
sha_chksum=None):
"""Creates a new Virtual Disk and uploads a data stream to it.
:param adapter: The adapter to talk over the API.
:param v_uuid: The Virtual I/O Server UUID that will host the disk.
:param vol_grp_uuid: The volume group that will host the Virtual Disk's
UUID.
:param d_stream: The data stream (either a file handle or stream) to
upload. Must have the 'read' method that returns a chunk
of bytes.
:param d_name: The name that should be given to the disk on the Virtual
I/O Server that will contain the file.
:param f_size: The size (in bytes) of the stream to be uploaded.
:param d_size: (OPTIONAL) The desired size of the new VDisk in bytes. If
omitted or smaller than f_size, it will be set to match
f_size.
:param sha_chksum: (OPTIONAL) The SHA256 checksum for the file. Useful for
integrity checks.
:return: The first return value is the virtual disk that the file is
uploaded into.
:return: Normally the second return value will be None, indicating that the
disk and image were uploaded without issue. If for some reason
the File metadata for the VIOS was not cleaned up, the return
value is the File EntryWrapper. This is simply a metadata marker
to be later used to retry the cleanup.
"""
# Create the new virtual disk. The size here is in GB. We can use decimal
# precision on the create call. What the VIOS will then do is determine
# the appropriate segment size (pp) and will provide a virtual disk that
# is 'at least' that big. Depends on the segment size set up on the
# volume group how much over it could go.
#
# See note below...temporary workaround needed.
if d_size is None or d_size < f_size:
d_size = f_size
gb_size = util.convert_bytes_to_gb(d_size)
# TODO(IBM) Temporary - need to round up to the highest GB. This should
# be done by the platform in the future.
gb_size = math.ceil(gb_size)
n_vdisk = crt_vdisk(adapter, v_uuid, vol_grp_uuid, d_name, gb_size)
# The file type. If local API server, then we can use the coordinated
# file path. Otherwise standard upload.
file_type = (vf.FileType.DISK_IMAGE_COORDINATED
if adapter.traits.local_api else vf.FileType.DISK_IMAGE)
# Next, create the file, but specify the appropriate disk udid from the
# Virtual Disk
vio_file = _create_file(adapter,
d_name,
file_type,
v_uuid,
f_size=f_size,
tdev_udid=n_vdisk.udid,
sha_chksum=sha_chksum)
maybe_file = _upload_stream(vio_file, d_stream)
return n_vdisk, maybe_file
