def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `tank.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `tank.store.ImageAddResult` object
:raises `tank.common.exception.Duplicate` if the image already
existed
"""
location = StoreLocation({'image': image_id})
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
logger.debug('creating image %s with order %d',
image_name, order)
try:
rbd.RBD().create(ioctx, image_name, image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(
_('RBD image %s already exists') % image_id)
with rbd.Image(ioctx, image_name) as image:
bytes_left = image_size
while bytes_left > 0:
length = min(self.chunk_size, bytes_left)
data = image_file.read(length)
image.write(data, image_size - bytes_left)
bytes_left -= length
checksum.update(data)
return (location.get_uri(), image_size, checksum.hexdigest())
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `tank.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `tank.store.ImageAddResult` object
:raises `tank.common.exception.Duplicate` if the image already
existed
"""
location = StoreLocation({'image': image_id})
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
logger.debug('creating image %s with order %d', image_name,
order)
try:
rbd.RBD().create(ioctx, image_name, image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(
_('RBD image %s already exists') % image_id)
with rbd.Image(ioctx, image_name) as image:
bytes_left = image_size
while bytes_left > 0:
length = min(self.chunk_size, bytes_left)
data = image_file.read(length)
image.write(data, image_size - bytes_left)
bytes_left -= length
checksum.update(data)
return (location.get_uri(), image_size, checksum.hexdigest())
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `tank.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `tank.store.ImageAddResult` object
:raises `tank.common.exception.Duplicate` if the image already
existed
Chase writes the image data using the scheme:
``chase://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
where:
<USER> = ``chase_store_user``
<KEY> = ``chase_store_key``
<AUTH_ADDRESS> = ``chase_store_auth_address``
<CONTAINER> = ``chase_store_container``
<ID> = The id of the image being added
:note Chase auth URLs by default use HTTPS. To specify an HTTP
auth URL, you can specify http://someurl.com for the
chase_store_auth_address config option
:note Chase cannot natively/transparently handle objects >5GB
in size. So, if the image is greater than 5GB, we write
chunks of image data to Chase and then write an manifest
to Chase that contains information about the chunks.
This same chunking process is used by default for images
of an unknown size, as pushing them directly to chase would
fail if the image turns out to be greater than 5GB.
"""
chase_conn = self._make_chase_connection(
auth_url=self.full_auth_address, user=self.user, key=self.key)
create_container_if_missing(self.container, chase_conn, self.conf)
obj_name = str(image_id)
location = StoreLocation({'scheme': self.scheme,
'container': self.container,
'obj': obj_name,
'authurl': self.auth_address,
'user': self.user,
'key': self.key})
logger.debug(_("Adding image object '%(obj_name)s' "
"to Chase") % locals())
try:
if image_size > 0 and image_size < self.large_object_size:
# Image size is known, and is less than large_object_size.
# Send to Chase with regular PUT.
obj_etag = chase_conn.put_object(self.container, obj_name,
image_file,
content_length=image_size)
else:
# Write the image into Chase in chunks.
chunk_id = 1
if image_size > 0:
total_chunks = str(int(
math.ceil(float(image_size) /
float(self.large_object_chunk_size))))
else:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size.
logger.debug(_("Cannot determine image size. Adding as a "
"segmented object to Chase."))
total_chunks = '?'
checksum = hashlib.md5()
combined_chunks_size = 0
while True:
chunk_size = self.large_object_chunk_size
if image_size == 0:
content_length = None
else:
left = image_size - combined_chunks_size
if left == 0:
break
if chunk_size > left:
chunk_size = left
content_length = chunk_size
chunk_name = "%s-%05d" % (obj_name, chunk_id)
reader = ChunkReader(image_file, checksum, chunk_size)
chunk_etag = chase_conn.put_object(
self.container, chunk_name, reader,
content_length=content_length)
bytes_read = reader.bytes_read
logger.debug(_("Wrote chunk %(chunk_id)d/"
"%(total_chunks)s of length %(bytes_read)d "
"to Chase returning MD5 of content: "
"%(chunk_etag)s")
% locals())
if bytes_read == 0:
# Delete the last chunk, because it's of zero size.
# This will happen if image_size == 0.
logger.debug(_("Deleting final zero-length chunk"))
chase_conn.delete_object(self.container, chunk_name)
break
chunk_id += 1
combined_chunks_size += bytes_read
# In the case we have been given an unknown image size,
# set the image_size to the total size of the combined chunks.
if image_size == 0:
image_size = combined_chunks_size
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s" % (self.container, obj_name)
headers = {'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
_ignored = chase_conn.put_object(self.container, obj_name,
None, headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
return (location.get_uri(), image_size, obj_etag)
except chase_client.ClientException, e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(_("Chase already has an image at "
"location %s") % location.get_uri())
msg = (_("Failed to add object to Chase.\n"
"Got error from Chase: %(e)s") % locals())
logger.error(msg)
raise tank.store.BackendException(msg)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `tank.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `tank.store.ImageAddResult` object
:raises `tank.common.exception.Duplicate` if the image already
existed
Chase writes the image data using the scheme:
``chase://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
where:
<USER> = ``chase_store_user``
<KEY> = ``chase_store_key``
<AUTH_ADDRESS> = ``chase_store_auth_address``
<CONTAINER> = ``chase_store_container``
<ID> = The id of the image being added
:note Chase auth URLs by default use HTTPS. To specify an HTTP
auth URL, you can specify http://someurl.com for the
chase_store_auth_address config option
:note Chase cannot natively/transparently handle objects >5GB
in size. So, if the image is greater than 5GB, we write
chunks of image data to Chase and then write an manifest
to Chase that contains information about the chunks.
This same chunking process is used by default for images
of an unknown size, as pushing them directly to chase would
fail if the image turns out to be greater than 5GB.
"""
chase_conn = self._make_chase_connection(
auth_url=self.full_auth_address, user=self.user, key=self.key)
create_container_if_missing(self.container, chase_conn, self.conf)
obj_name = str(image_id)
location = StoreLocation({
'scheme': self.scheme,
'container': self.container,
'obj': obj_name,
'authurl': self.auth_address,
'user': self.user,
'key': self.key
})
logger.debug(
_("Adding image object '%(obj_name)s' "
"to Chase") % locals())
try:
if image_size > 0 and image_size < self.large_object_size:
# Image size is known, and is less than large_object_size.
# Send to Chase with regular PUT.
obj_etag = chase_conn.put_object(self.container,
obj_name,
image_file,
content_length=image_size)
else:
# Write the image into Chase in chunks.
chunk_id = 1
if image_size > 0:
total_chunks = str(
int(
math.ceil(
float(image_size) /
float(self.large_object_chunk_size))))
else:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size.
logger.debug(
_("Cannot determine image size. Adding as a "
"segmented object to Chase."))
total_chunks = '?'
checksum = hashlib.md5()
combined_chunks_size = 0
while True:
chunk_size = self.large_object_chunk_size
if image_size == 0:
content_length = None
else:
left = image_size - combined_chunks_size
if left == 0:
break
if chunk_size > left:
chunk_size = left
content_length = chunk_size
chunk_name = "%s-%05d" % (obj_name, chunk_id)
reader = ChunkReader(image_file, checksum, chunk_size)
chunk_etag = chase_conn.put_object(
self.container,
chunk_name,
reader,
content_length=content_length)
bytes_read = reader.bytes_read
logger.debug(
_("Wrote chunk %(chunk_id)d/"
"%(total_chunks)s of length %(bytes_read)d "
"to Chase returning MD5 of content: "
"%(chunk_etag)s") % locals())
if bytes_read == 0:
# Delete the last chunk, because it's of zero size.
# This will happen if image_size == 0.
logger.debug(_("Deleting final zero-length chunk"))
chase_conn.delete_object(self.container, chunk_name)
break
chunk_id += 1
combined_chunks_size += bytes_read
# In the case we have been given an unknown image size,
# set the image_size to the total size of the combined chunks.
if image_size == 0:
image_size = combined_chunks_size
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s" % (self.container, obj_name)
headers = {
'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest
}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
_ignored = chase_conn.put_object(self.container,
obj_name,
None,
headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
return (location.get_uri(), image_size, obj_etag)
except chase_client.ClientException, e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(
_("Chase already has an image at "
"location %s") % location.get_uri())
msg = (_("Failed to add object to Chase.\n"
"Got error from Chase: %(e)s") % locals())
logger.error(msg)
raise tank.store.BackendException(msg)