def schedule_delete_from_backend(uri, conf, context, image_id, **kwargs):
"""
Given a uri and a time, schedule the deletion of an image.
"""
conf.register_opts(delete_opts)
if not conf.delayed_delete:
registry.update_image_metadata(context, image_id,
{'status': 'deleted'})
try:
return delete_from_backend(uri, **kwargs)
except (UnsupportedBackend, exception.NotFound):
msg = _("Failed to delete image from store (%(uri)s).") % locals()
logger.error(msg)
datadir = get_scrubber_datadir(conf)
delete_time = time.time() + conf.scrub_time
file_path = os.path.join(datadir, str(image_id))
utils.safe_mkdirs(datadir)
if os.path.exists(file_path):
msg = _("Image id %(image_id)s already queued for delete") % {
'image_id': image_id
}
raise exception.Duplicate(msg)
with open(file_path, 'w') as f:
f.write('\n'.join([uri, str(int(delete_time))]))
os.chmod(file_path, 0600)
os.utime(file_path, (delete_time, delete_time))
registry.update_image_metadata(context, image_id,
{'status': 'pending_delete'})
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `tank.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `tank.store.ImageAddResult` object
:raises `tank.common.exception.Duplicate` if the image already
existed
:note By default, the backend writes the image data to a file
`/<DATADIR>/<ID>`, where <DATADIR> is the value of
the filesystem_store_datadir configuration option and <ID>
is the supplied image ID.
"""
filepath = os.path.join(self.datadir, str(image_id))
if os.path.exists(filepath):
raise exception.Duplicate(
_("Image file %s already exists!") % filepath)
checksum = hashlib.md5()
bytes_written = 0
with open(filepath, 'wb') as f:
while True:
buf = image_file.read(ChunkedFile.CHUNKSIZE)
if not buf:
break
bytes_written += len(buf)
checksum.update(buf)
f.write(buf)
checksum_hex = checksum.hexdigest()
logger.debug(
_("Wrote %(bytes_written)d bytes to %(filepath)s with "
"checksum %(checksum_hex)s") % locals())
return ('file://%s' % filepath, bytes_written, checksum_hex)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `tank.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `tank.store.ImageAddResult` object
:raises `tank.common.exception.Duplicate` if the image already
existed
"""
location = StoreLocation({'image': image_id})
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
logger.debug('creating image %s with order %d', image_name,
order)
try:
rbd.RBD().create(ioctx, image_name, image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(
_('RBD image %s already exists') % image_id)
with rbd.Image(ioctx, image_name) as image:
bytes_left = image_size
while bytes_left > 0:
length = min(self.chunk_size, bytes_left)
data = image_file.read(length)
image.write(data, image_size - bytes_left)
bytes_left -= length
checksum.update(data)
return (location.get_uri(), image_size, checksum.hexdigest())
def _image_update(context, values, image_id, purge_props=False):
"""
Used internally by image_create and image_update
:param context: Request context
:param values: A dict of attributes to set
:param image_id: If None, create the image, otherwise, find and update it
"""
session = get_session()
with session.begin():
# Remove the properties passed in the values mapping. We
# handle properties separately from base image attributes,
# and leaving properties in the values mapping will cause
# a SQLAlchemy model error because SQLAlchemy expects the
# properties attribute of an Image model to be a list and
# not a dict.
properties = values.pop('properties', {})
if image_id:
image_ref = image_get(context, image_id, session=session)
# Perform authorization check
check_mutate_authorization(context, image_ref)
else:
if 'size' in values:
values['size'] = int(values['size'])
if 'min_ram' in values:
values['min_ram'] = int(values['min_ram'] or 0)
if 'min_disk' in values:
values['min_disk'] = int(values['min_disk'] or 0)
values['is_public'] = bool(values.get('is_public', False))
values['protected'] = bool(values.get('protected', False))
image_ref = models.Image()
# Need to canonicalize ownership
if 'owner' in values and not values['owner']:
values['owner'] = None
if image_id:
# Don't drop created_at if we're passing it in...
_drop_protected_attrs(models.Image, values)
image_ref.update(values)
# Validate the attributes before we go any further. From my
# investigation, the @validates decorator does not validate
# on new records, only on existing records, which is, well,
# idiotic.
validate_image(image_ref.to_dict())
try:
image_ref.save(session=session)
except IntegrityError, e:
raise exception.Duplicate("Image ID %s already exists!" %
values['id'])
_set_properties_for_image(context, image_ref, properties, purge_props,
session)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `tank.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `tank.store.ImageAddResult` object
:raises `tank.common.exception.Duplicate` if the image already
existed
Chase writes the image data using the scheme:
``chase://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
where:
<USER> = ``chase_store_user``
<KEY> = ``chase_store_key``
<AUTH_ADDRESS> = ``chase_store_auth_address``
<CONTAINER> = ``chase_store_container``
<ID> = The id of the image being added
:note Chase auth URLs by default use HTTPS. To specify an HTTP
auth URL, you can specify http://someurl.com for the
chase_store_auth_address config option
:note Chase cannot natively/transparently handle objects >5GB
in size. So, if the image is greater than 5GB, we write
chunks of image data to Chase and then write an manifest
to Chase that contains information about the chunks.
This same chunking process is used by default for images
of an unknown size, as pushing them directly to chase would
fail if the image turns out to be greater than 5GB.
"""
chase_conn = self._make_chase_connection(
auth_url=self.full_auth_address, user=self.user, key=self.key)
create_container_if_missing(self.container, chase_conn, self.conf)
obj_name = str(image_id)
location = StoreLocation({
'scheme': self.scheme,
'container': self.container,
'obj': obj_name,
'authurl': self.auth_address,
'user': self.user,
'key': self.key
})
logger.debug(
_("Adding image object '%(obj_name)s' "
"to Chase") % locals())
try:
if image_size > 0 and image_size < self.large_object_size:
# Image size is known, and is less than large_object_size.
# Send to Chase with regular PUT.
obj_etag = chase_conn.put_object(self.container,
obj_name,
image_file,
content_length=image_size)
else:
# Write the image into Chase in chunks.
chunk_id = 1
if image_size > 0:
total_chunks = str(
int(
math.ceil(
float(image_size) /
float(self.large_object_chunk_size))))
else:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size.
logger.debug(
_("Cannot determine image size. Adding as a "
"segmented object to Chase."))
total_chunks = '?'
checksum = hashlib.md5()
combined_chunks_size = 0
while True:
chunk_size = self.large_object_chunk_size
if image_size == 0:
content_length = None
else:
left = image_size - combined_chunks_size
if left == 0:
break
if chunk_size > left:
chunk_size = left
content_length = chunk_size
chunk_name = "%s-%05d" % (obj_name, chunk_id)
reader = ChunkReader(image_file, checksum, chunk_size)
chunk_etag = chase_conn.put_object(
self.container,
chunk_name,
reader,
content_length=content_length)
bytes_read = reader.bytes_read
logger.debug(
_("Wrote chunk %(chunk_id)d/"
"%(total_chunks)s of length %(bytes_read)d "
"to Chase returning MD5 of content: "
"%(chunk_etag)s") % locals())
if bytes_read == 0:
# Delete the last chunk, because it's of zero size.
# This will happen if image_size == 0.
logger.debug(_("Deleting final zero-length chunk"))
chase_conn.delete_object(self.container, chunk_name)
break
chunk_id += 1
combined_chunks_size += bytes_read
# In the case we have been given an unknown image size,
# set the image_size to the total size of the combined chunks.
if image_size == 0:
image_size = combined_chunks_size
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s" % (self.container, obj_name)
headers = {
'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest
}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
_ignored = chase_conn.put_object(self.container,
obj_name,
None,
headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
return (location.get_uri(), image_size, obj_etag)
except chase_client.ClientException, e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(
_("Chase already has an image at "
"location %s") % location.get_uri())
msg = (_("Failed to add object to Chase.\n"
"Got error from Chase: %(e)s") % locals())
logger.error(msg)
raise tank.store.BackendException(msg)
def _do_request(self, method, url, body, headers):
"""
Connects to the server and issues a request. Handles converting
any returned HTTP error status codes to X7/Tank exceptions
and closing the server connection. Returns the result data, or
raises an appropriate exception.
:param method: HTTP method ("GET", "POST", "PUT", etc...)
:param url: urlparse.ParsedResult object with URL information
:param body: string of data to send, or None (default)
:param headers: mapping of key/value pairs to add as headers
:note
If the body param has a read attribute, and method is either
POST or PUT, this method will automatically conduct a chunked-transfer
encoding and use the body as a file object, transferring chunks
of data using the connection's send() method. This allows large
objects to be transferred efficiently without buffering the entire
body in memory.
"""
if url.query:
path = url.path + "?" + url.query
else:
path = url.path
try:
connection_type = self.get_connection_type()
headers = headers or {}
if 'x-auth-token' not in headers and self.auth_tok:
headers['x-auth-token'] = self.auth_tok
c = connection_type(url.hostname, url.port, **self.connect_kwargs)
# Do a simple request or a chunked request, depending
# on whether the body param is a file-like object and
# the method is PUT or POST
if hasattr(body, 'read') and method.lower() in ('post', 'put'):
# Chunk it, baby...
c.putrequest(method, path)
for header, value in headers.items():
c.putheader(header, value)
c.putheader('Transfer-Encoding', 'chunked')
c.endheaders()
chunk = body.read(self.CHUNKSIZE)
while chunk:
c.send('%x\r\n%s\r\n' % (len(chunk), chunk))
chunk = body.read(self.CHUNKSIZE)
c.send('0\r\n\r\n')
else:
# Simple request...
c.request(method, path, body, headers)
res = c.getresponse()
status_code = self.get_status_code(res)
if status_code in self.OK_RESPONSE_CODES:
return res
elif status_code in self.REDIRECT_RESPONSE_CODES:
raise exception.RedirectException(res.getheader('Location'))
elif status_code == httplib.UNAUTHORIZED:
raise exception.NotAuthorized(res.read())
elif status_code == httplib.FORBIDDEN:
raise exception.NotAuthorized(res.read())
elif status_code == httplib.NOT_FOUND:
raise exception.NotFound(res.read())
elif status_code == httplib.CONFLICT:
raise exception.Duplicate(res.read())
elif status_code == httplib.BAD_REQUEST:
raise exception.Invalid(res.read())
elif status_code == httplib.MULTIPLE_CHOICES:
raise exception.MultipleChoices(body=res.read())
elif status_code == httplib.INTERNAL_SERVER_ERROR:
raise Exception("Internal Server error: %s" % res.read())
else:
raise Exception("Unknown error occurred! %s" % res.read())
except (socket.error, IOError), e:
raise exception.ClientConnectionError(e)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `tank.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `tank.store.ImageAddResult` object
:raises `tank.common.exception.Duplicate` if the image already
existed
S3 writes the image data using the scheme:
s3://<ACCESS_KEY>:<SECRET_KEY>@<S3_URL>/<BUCKET>/<OBJ>
where:
<USER> = ``s3_store_user``
<KEY> = ``s3_store_key``
<S3_HOST> = ``s3_store_host``
<BUCKET> = ``s3_store_bucket``
<ID> = The id of the image being added
"""
from boto.s3.connection import S3Connection
loc = StoreLocation({
'scheme': self.scheme,
'bucket': self.bucket,
'key': image_id,
's3serviceurl': self.full_s3_host,
'accesskey': self.access_key,
'secretkey': self.secret_key
})
s3_conn = S3Connection(loc.accesskey,
loc.secretkey,
host=loc.s3serviceurl,
is_secure=(loc.scheme == 's3+https'))
create_bucket_if_missing(self.bucket, s3_conn, self.conf)
bucket_obj = get_bucket(s3_conn, self.bucket)
obj_name = str(image_id)
key = bucket_obj.get_key(obj_name)
if key and key.exists():
raise exception.Duplicate(
_("S3 already has an image at "
"location %s") % loc.get_uri())
msg = _("Adding image object to S3 using (s3_host=%(s3_host)s, "
"access_key=%(access_key)s, bucket=%(bucket)s, "
"key=%(obj_name)s)") % ({
's3_host': self.s3_host,
'access_key': self.access_key,
'bucket': self.bucket,
'obj_name': obj_name
})
logger.debug(msg)
key = bucket_obj.new_key(obj_name)
# We need to wrap image_file, which is a reference to the
# webob.Request.body_file, with a seekable file-like object,
# otherwise the call to set_contents_from_file() will die
# with an error about Input object has no method 'seek'. We
# might want to call webob.Request.make_body_seekable(), but
# unfortunately, that method copies the entire image into
# memory and results in LP Bug #818292 occurring. So, here
# we write temporary file in as memory-efficient manner as
# possible and then supply the temporary file to S3. We also
# take this opportunity to calculate the image checksum while
# writing the tempfile, so we don't need to call key.compute_md5()
msg = _("Writing request body file to temporary file "
"for %s") % loc.get_uri()
logger.debug(msg)
tmpdir = self.s3_store_object_buffer_dir
temp_file = tempfile.NamedTemporaryFile(dir=tmpdir)
checksum = hashlib.md5()
chunk = image_file.read(self.CHUNKSIZE)
while chunk:
checksum.update(chunk)
temp_file.write(chunk)
chunk = image_file.read(self.CHUNKSIZE)
temp_file.flush()
msg = _("Uploading temporary file to S3 for %s") % loc.get_uri()
logger.debug(msg)
# OK, now upload the data into the key
key.set_contents_from_file(open(temp_file.name, 'r+b'), replace=False)
size = key.size
checksum_hex = checksum.hexdigest()
logger.debug(
_("Wrote %(size)d bytes to S3 key named %(obj_name)s "
"with checksum %(checksum_hex)s") % locals())
return (loc.get_uri(), size, checksum_hex)