def add_image_file(self, full_data_path, image_file):
"""
Add image file or return exception
"""
try:
LOG.debug("attempting to create image file in irods '%s'" %
full_data_path)
file_object = self.irods_conn_object.data_objects.create(
full_data_path)
except:
LOG.error("file with same name exists in the same path")
raise exceptions.Duplicate(
_("image file %s already exists " + "or no perms") % filepath)
LOG.debug("performing the write")
checksum = hashlib.md5()
bytes_written = 0
try:
with file_object.open('r+') as f:
for buf in utils.chunkreadable(image_file,
ChunkedFile.default_chunk_size):
bytes_written += len(buf)
checksum.update(buf)
f.write(buf)
except Exception as e:
# let's attempt an delete
file_object.unlink()
reason = _('paritial write, transfer failed')
LOG.error(e)
raise exceptions.StorageWriteDenied(reason)
finally:
self.irods_conn_object.cleanup()
file_object = None
checksum_hex = checksum.hexdigest()
LOG.debug(
_("Wrote %(bytes_written)d bytes to %(full_data_path)s, "
"checksum = %(checksum_hex)s") % locals())
return [bytes_written, checksum_hex]
def add(self, image_id, image_file, image_size, hashing_algo, context=None,
verifier=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:param hashing_algo: A hashlib algorithm identifier (string)
:param context: The request context
:param verifier: An object used to verify signatures for images
:returns: tuple of: (1) URL in backing store, (2) bytes written,
(3) checksum, (4) multihash value, and (5) a dictionary
with storage system specific information
:raises: `glance_store.exceptions.Duplicate` if the image already
exists
:note:: By default, the backend writes the image data to a file
`/<DATADIR>/<ID>`, where <DATADIR> is the value of
the filesystem_store_datadir configuration option and <ID>
is the supplied image ID.
"""
datadir = self._find_best_datadir(image_size)
filepath = os.path.join(datadir, str(image_id))
if os.path.exists(filepath):
raise exceptions.Duplicate(image=filepath)
os_hash_value = hashlib.new(str(hashing_algo))
checksum = hashlib.md5()
bytes_written = 0
try:
with open(filepath, 'wb') as f:
for buf in utils.chunkreadable(image_file,
self.WRITE_CHUNKSIZE):
bytes_written += len(buf)
os_hash_value.update(buf)
checksum.update(buf)
if verifier:
verifier.update(buf)
f.write(buf)
except IOError as e:
if e.errno != errno.EACCES:
self._delete_partial(filepath, image_id)
errors = {errno.EFBIG: exceptions.StorageFull(),
errno.ENOSPC: exceptions.StorageFull(),
errno.EACCES: exceptions.StorageWriteDenied()}
raise errors.get(e.errno, e)
except Exception:
with excutils.save_and_reraise_exception():
self._delete_partial(filepath, image_id)
hash_hex = os_hash_value.hexdigest()
checksum_hex = checksum.hexdigest()
metadata = self._get_metadata(filepath)
LOG.debug(("Wrote %(bytes_written)d bytes to %(filepath)s with "
"checksum %(checksum_hex)s and multihash %(hash_hex)s"),
{'bytes_written': bytes_written,
'filepath': filepath,
'checksum_hex': checksum_hex,
'hash_hex': hash_hex})
if self.backend_group:
fstore_perm = getattr(
self.conf, self.backend_group).filesystem_store_file_perm
else:
fstore_perm = self.conf.glance_store.filesystem_store_file_perm
if fstore_perm > 0:
perm = int(str(fstore_perm), 8)
try:
os.chmod(filepath, perm)
except (IOError, OSError):
LOG.warning(_LW("Unable to set permission to image: %s") %
filepath)
# Add store backend information to location metadata
if self.backend_group:
metadata['backend'] = u"%s" % self.backend_group
return ('file://%s' % filepath,
bytes_written,
checksum_hex,
hash_hex,
metadata)
def add(self,
image_id,
image_file,
image_size,
hashing_algo,
context=None,
verifier=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:param hashing_algo: A hashlib algorithm identifier (string)
:param context: The request context
:param verifier: An object used to verify signatures for images
:returns: tuple of: (1) URL in backing store, (2) bytes written,
(3) checksum, (4) multihash value, and (5) a dictionary
with storage system specific information
:raises: `glance_store.exceptions.Duplicate` if the image already
exists
"""
self._check_context(context, require_tenant=True)
client = self.get_cinderclient(context)
os_hash_value = utils.get_hasher(hashing_algo, False)
checksum = utils.get_hasher('md5', False)
bytes_written = 0
size_gb = int(math.ceil(float(image_size) / units.Gi))
if size_gb == 0:
size_gb = 1
name = "image-%s" % image_id
owner = context.project_id
metadata = {
'glance_image_id': image_id,
'image_size': str(image_size),
'image_owner': owner
}
volume_type = self.store_conf.cinder_volume_type
LOG.debug('Creating a new volume: image_size=%d size_gb=%d type=%s',
image_size, size_gb, volume_type or 'None')
if image_size == 0:
LOG.info(
_LI("Since image size is zero, we will be doing "
"resize-before-write for each GB which "
"will be considerably slower than normal."))
volume = client.volumes.create(size_gb,
name=name,
metadata=metadata,
volume_type=volume_type)
volume = self._wait_volume_status(volume, 'creating', 'available')
size_gb = volume.size
failed = True
need_extend = True
buf = None
try:
while need_extend:
with self._open_cinder_volume(client, volume, 'wb') as f:
f.seek(bytes_written)
if buf:
f.write(buf)
bytes_written += len(buf)
while True:
buf = image_file.read(self.WRITE_CHUNKSIZE)
if not buf:
need_extend = False
break
os_hash_value.update(buf)
checksum.update(buf)
if verifier:
verifier.update(buf)
if (bytes_written + len(buf) > size_gb * units.Gi
and image_size == 0):
break
f.write(buf)
bytes_written += len(buf)
if need_extend:
size_gb += 1
LOG.debug("Extending volume %(volume_id)s to %(size)s GB.",
{
'volume_id': volume.id,
'size': size_gb
})
volume.extend(volume, size_gb)
try:
volume = self._wait_volume_status(
volume, 'extending', 'available')
size_gb = volume.size
except exceptions.BackendException:
raise exceptions.StorageFull()
failed = False
except IOError as e:
# Convert IOError reasons to Glance Store exceptions
errors = {
errno.EFBIG: exceptions.StorageFull(),
errno.ENOSPC: exceptions.StorageFull(),
errno.EACCES: exceptions.StorageWriteDenied()
}
raise errors.get(e.errno, e)
finally:
if failed:
LOG.error(_LE("Failed to write to volume %(volume_id)s."),
{'volume_id': volume.id})
try:
volume.delete()
except Exception:
LOG.exception(
_LE('Failed to delete of volume '
'%(volume_id)s.'), {'volume_id': volume.id})
if image_size == 0:
metadata.update({'image_size': str(bytes_written)})
volume.update_all_metadata(metadata)
volume.update_readonly_flag(volume, True)
hash_hex = os_hash_value.hexdigest()
checksum_hex = checksum.hexdigest()
LOG.debug(
"Wrote %(bytes_written)d bytes to volume %(volume_id)s "
"with checksum %(checksum_hex)s.", {
'bytes_written': bytes_written,
'volume_id': volume.id,
'checksum_hex': checksum_hex
})
image_metadata = {}
location_url = 'cinder://%s' % volume.id
if self.backend_group:
image_metadata['store'] = u"%s" % self.backend_group
location_url = 'cinder://%s/%s' % (self.backend_group, volume.id)
return (location_url, bytes_written, checksum_hex, hash_hex,
image_metadata)
def add(self, image_id, image_file, image_size, context=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance_store.exceptions.Duplicate` if the image already
existed
:note By default, the backend writes the image data to a file
`/<DATADIR>/<ID>`, where <DATADIR> is the value of
the filesystem_store_datadir configuration option and <ID>
is the supplied image ID.
"""
datadir = self._find_best_datadir(image_size)
filepath = os.path.join(datadir, str(image_id))
if os.path.exists(filepath):
raise exceptions.Duplicate(image=filepath)
checksum = hashlib.md5()
bytes_written = 0
try:
with open(filepath, 'wb') as f:
for buf in utils.chunkreadable(image_file,
self.WRITE_CHUNKSIZE):
bytes_written += len(buf)
checksum.update(buf)
f.write(buf)
except IOError as e:
if e.errno != errno.EACCES:
self._delete_partial(filepath, image_id)
errors = {errno.EFBIG: exceptions.StorageFull(),
errno.ENOSPC: exceptions.StorageFull(),
errno.EACCES: exceptions.StorageWriteDenied()}
raise errors.get(e.errno, e)
except Exception:
with excutils.save_and_reraise_exception():
self._delete_partial(filepath, image_id)
checksum_hex = checksum.hexdigest()
metadata = self._get_metadata(filepath)
LOG.debug(_("Wrote %(bytes_written)d bytes to %(filepath)s with "
"checksum %(checksum_hex)s"),
{'bytes_written': bytes_written,
'filepath': filepath,
'checksum_hex': checksum_hex})
if self.conf.glance_store.filesystem_store_file_perm > 0:
perm = int(str(self.conf.glance_store.filesystem_store_file_perm),
8)
try:
os.chmod(filepath, perm)
except (IOError, OSError):
LOG.warn(_LW("Unable to set permission to image: %s") %
filepath)
return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)
def add(self,
image_id,
image_file,
image_size,
context=None,
verifier=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:param context: The request context
:param verifier: An object used to verify signatures for images
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance_store.exceptions.Duplicate` if the image already
existed
"""
self._check_context(context, require_tenant=True)
client = get_cinderclient(self.conf, context)
checksum = hashlib.md5()
bytes_written = 0
size_gb = int((image_size + units.Gi - 1) / units.Gi)
if size_gb == 0:
size_gb = 1
name = "image-%s" % image_id
owner = context.tenant
metadata = {
'glance_image_id': image_id,
'image_size': str(image_size),
'image_owner': owner
}
LOG.debug('Creating a new volume: image_size=%d size_gb=%d',
image_size, size_gb)
if image_size == 0:
LOG.info(
_LI("Since image size is zero, we will be doing "
"resize-before-write for each GB which "
"will be considerably slower than normal."))
volume = client.volumes.create(size_gb, name=name, metadata=metadata)
volume = self._wait_volume_status(volume, 'creating', 'available')
failed = True
need_extend = True
buf = None
try:
while need_extend:
with self._open_cinder_volume(client, volume, 'wb') as f:
f.seek(bytes_written)
if buf:
f.write(buf)
bytes_written += len(buf)
while True:
buf = image_file.read(self.WRITE_CHUNKSIZE)
if not buf:
need_extend = False
break
checksum.update(buf)
if verifier:
verifier.update(buf)
if (bytes_written + len(buf) > size_gb * units.Gi
and image_size == 0):
break
f.write(buf)
bytes_written += len(buf)
if need_extend:
size_gb += 1
LOG.debug("Extending volume %(volume_id)s to %(size)s GB.",
{
'volume_id': volume.id,
'size': size_gb
})
volume.extend(volume, size_gb)
try:
volume = self._wait_volume_status(
volume, 'extending', 'available')
except exceptions.BackendException:
raise exceptions.StorageFull()
failed = False
except IOError as e:
# Convert IOError reasons to Glance Store exceptions
errors = {
errno.EFBIG: exceptions.StorageFull(),
errno.ENOSPC: exceptions.StorageFull(),
errno.EACCES: exceptions.StorageWriteDenied()
}
raise errors.get(e.errno, e)
finally:
if failed:
LOG.error(_LE("Failed to write to volume %(volume_id)s."),
{'volume_id': volume.id})
try:
volume.delete()
except Exception:
LOG.exception(
_LE('Failed to delete of volume '
'%(volume_id)s.'), {'volume_id': volume.id})
if image_size == 0:
metadata.update({'image_size': str(bytes_written)})
volume.update_all_metadata(metadata)
volume.update_readonly_flag(volume, True)
checksum_hex = checksum.hexdigest()
LOG.debug(
"Wrote %(bytes_written)d bytes to volume %(volume_id)s "
"with checksum %(checksum_hex)s.", {
'bytes_written': bytes_written,
'volume_id': volume.id,
'checksum_hex': checksum_hex
})
return ('cinder://%s' % volume.id, bytes_written, checksum_hex, {})
