def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed :note By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ datadir = self._find_best_datadir(image_size) filepath = os.path.join(datadir, str(image_id)) if os.path.exists(filepath): raise exception.Duplicate( _("Image file %s already exists!") % filepath) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, ChunkedFile.CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) except IOError as e: if e.errno != errno.EACCES: self._delete_partial(filepath, image_id) exceptions = { errno.EFBIG: exception.StorageFull(), errno.ENOSPC: exception.StorageFull(), errno.EACCES: exception.StorageWriteDenied() } raise exceptions.get(e.errno, e) except Exception: with excutils.save_and_reraise_exception(): self._delete_partial(filepath, image_id) checksum_hex = checksum.hexdigest() metadata = self._get_metadata() LOG.debug( _("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s"), { 'bytes_written': bytes_written, 'filepath': filepath, 'checksum_hex': checksum_hex }) return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)
def image_iterator(self, connection, headers, body): if self._sendable(body): return SendFileIterator(connection, body) elif self._iterable(body): return utils.chunkreadable(body) else: return ImageBodyIterator(body)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed :note By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ datadir = self._find_best_datadir(image_size) filepath = os.path.join(datadir, str(image_id)) if os.path.exists(filepath): raise exception.Duplicate(_("Image file %s already exists!") % filepath) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, ChunkedFile.CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) except IOError as e: if e.errno != errno.EACCES: self._delete_partial(filepath, image_id) exceptions = {errno.EFBIG: exception.StorageFull(), errno.ENOSPC: exception.StorageFull(), errno.EACCES: exception.StorageWriteDenied()} raise exceptions.get(e.errno, e) except Exception: with excutils.save_and_reraise_exception(): self._delete_partial(filepath, image_id) checksum_hex = checksum.hexdigest() metadata = self._get_metadata() LOG.debug("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s", {'bytes_written': bytes_written, 'filepath': filepath, 'checksum_hex': checksum_hex}) return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns an `glance.store.ImageAddResult` object containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval `glance.store.ImageAddResult` object :raises `glance.common.exception.Duplicate` if the image already existed :note By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ filepath = os.path.join(self.datadir, str(image_id)) if os.path.exists(filepath): raise exception.Duplicate( _("Image file %s already exists!") % filepath) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, ChunkedFile.CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) except IOError as e: if e.errno in [errno.EFBIG, errno.ENOSPC]: try: os.unlink(filepath) except Exception: msg = _('Unable to remove partial image data for image %s') LOG.error(msg % image_id) raise exception.StorageFull() elif e.errno == errno.EACCES: raise exception.StorageWriteDenied() else: raise checksum_hex = checksum.hexdigest() LOG.debug( _("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s") % locals()) return ('file://%s' % filepath, bytes_written, checksum_hex)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns an `glance.store.ImageAddResult` object containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval `glance.store.ImageAddResult` object :raises `glance.common.exception.Duplicate` if the image already existed :note By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ filepath = os.path.join(self.datadir, str(image_id)) if os.path.exists(filepath): raise exception.Duplicate(_("Image file %s already exists!") % filepath) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, ChunkedFile.CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) except IOError as e: if e.errno in [errno.EFBIG, errno.ENOSPC]: try: os.unlink(filepath) except Exception: msg = _('Unable to remove partial image data for image %s') LOG.error(msg % image_id) raise exception.StorageFull() elif e.errno == errno.EACCES: raise exception.StorageWriteDenied() else: raise checksum_hex = checksum.hexdigest() LOG.debug(_("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s") % locals()) return ('file://%s' % filepath, bytes_written, checksum_hex)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ checksum = hashlib.md5() image_name = str(image_id) with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn: fsid = None if hasattr(conn, 'get_fsid'): fsid = conn.get_fsid() with conn.open_ioctx(self.pool) as ioctx: order = int(math.log(self.chunk_size, 2)) LOG.debug('creating image %s with order %d', image_name, order) try: loc = self._create_image(fsid, ioctx, image_name, image_size, order) except rbd.ImageExists: raise exception.Duplicate( _('RBD image %s already exists') % image_id) try: with rbd.Image(ioctx, image_name) as image: offset = 0 chunks = utils.chunkreadable(image_file, self.chunk_size) for chunk in chunks: offset += image.write(chunk, offset) checksum.update(chunk) if loc.snapshot: image.create_snap(loc.snapshot) image.protect_snap(loc.snapshot) except: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exception. with excutils.save_and_reraise_exception(): self._delete_image(loc.image, loc.snapshot) return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed S3 writes the image data using the scheme: s3://<ACCESS_KEY>:<SECRET_KEY>@<S3_URL>/<BUCKET>/<OBJ> where: <USER> = ``s3_store_user`` <KEY> = ``s3_store_key`` <S3_HOST> = ``s3_store_host`` <BUCKET> = ``s3_store_bucket`` <ID> = The id of the image being added """ from boto.s3.connection import S3Connection loc = StoreLocation({ 'scheme': self.scheme, 'bucket': self.bucket, 'key': image_id, 's3serviceurl': self.full_s3_host, 'accesskey': self.access_key, 'secretkey': self.secret_key }) s3_conn = S3Connection(loc.accesskey, loc.secretkey, host=loc.s3serviceurl, is_secure=(loc.scheme == 's3+https'), calling_format=get_calling_format()) create_bucket_if_missing(self.bucket, s3_conn) bucket_obj = get_bucket(s3_conn, self.bucket) obj_name = str(image_id) def _sanitize(uri): return re.sub('//.*:.*@', '//s3_store_secret_key:s3_store_access_key@', uri) key = bucket_obj.get_key(obj_name) if key and key.exists(): raise exception.Duplicate( _("S3 already has an image at " "location %s") % _sanitize(loc.get_uri())) msg = ("Adding image object to S3 using (s3_host=%(s3_host)s, " "access_key=%(access_key)s, bucket=%(bucket)s, " "key=%(obj_name)s)" % ({ 's3_host': self.s3_host, 'access_key': self.access_key, 'bucket': self.bucket, 'obj_name': obj_name })) LOG.debug(msg) LOG.debug("Uploading an image file to S3 for %s" % _sanitize(loc.get_uri())) if image_size < self.s3_store_large_object_size: key = bucket_obj.new_key(obj_name) # We need to wrap image_file, which is a reference to the # webob.Request.body_file, with a seekable file-like object, # otherwise the call to set_contents_from_file() will die # with an error about Input object has no method 'seek'. We # might want to call webob.Request.make_body_seekable(), but # unfortunately, that method copies the entire image into # memory and results in LP Bug #818292 occurring. So, here # we write temporary file in as memory-efficient manner as # possible and then supply the temporary file to S3. We also # take this opportunity to calculate the image checksum while # writing the tempfile, so we don't need to call key.compute_md5() msg = ("Writing request body file to temporary file " "for %s") % _sanitize(loc.get_uri()) LOG.debug(msg) tmpdir = self.s3_store_object_buffer_dir temp_file = tempfile.NamedTemporaryFile(dir=tmpdir) checksum = hashlib.md5() for chunk in utils.chunkreadable(image_file, self.CHUNKSIZE): checksum.update(chunk) temp_file.write(chunk) temp_file.flush() msg = ("Uploading temporary file to S3 " "for %s") % _sanitize(loc.get_uri()) LOG.debug(msg) # OK, now upload the data into the key key.set_contents_from_file(open(temp_file.name, 'rb'), replace=False) size = key.size checksum_hex = checksum.hexdigest() LOG.debug("Wrote %(size)d bytes to S3 key named %(obj_name)s " "with checksum %(checksum_hex)s" % { 'size': size, 'obj_name': obj_name, 'checksum_hex': checksum_hex }) return (loc.get_uri(), size, checksum_hex, {}) else: checksum = hashlib.md5() parts = int( math.ceil( float(image_size) / float(self.s3_store_large_object_chunk_size))) threads = parts pool_size = CONF.s3_store_thread_pools pool = eventlet.greenpool.GreenPool(size=pool_size) mpu = bucket_obj.initiate_multipart_upload(obj_name) LOG.debug("Multipart initiate key=%(obj_name)s, " "UploadId=%(UploadId)s" % { 'obj_name': obj_name, 'UploadId': mpu.id }) cstart = 0 plist = [] it = utils.chunkreadable(image_file, self.s3_store_large_object_chunk_size) for p in range(threads): chunk = next(it) clen = len(chunk) checksum.update(chunk) fp = six.BytesIO(chunk) fp.seek(0) part = UploadPart(mpu, fp, cstart + 1, clen) pool.spawn_n(run_upload, part) plist.append(part) cstart += 1 pedict = {} total_size = 0 pool.waitall() for part in plist: pedict.update(part.etag) total_size += part.size success = True for part in plist: if not part.success: success = False if success: # Complete xml = get_mpu_xml(pedict) bucket_obj.complete_multipart_upload(obj_name, mpu.id, xml) checksum_hex = checksum.hexdigest() LOG.info( _LI("Multipart complete key=%(obj_name)s " "UploadId=%(UploadId)s " "Wrote %(total_size)d bytes to S3 key" "named %(obj_name)s " "with checksum %(checksum_hex)s") % { 'obj_name': obj_name, 'UploadId': mpu.id, 'total_size': total_size, 'obj_name': obj_name, 'checksum_hex': checksum_hex }) return (loc.get_uri(), total_size, checksum_hex, {}) else: # Abort bucket_obj.cancel_multipart_upload(obj_name, mpu.id) LOG.error( _LE("Some parts failed to upload to S3. " "Aborted the object key=%(obj_name)s") % {'obj_name': obj_name}) msg = (_("Failed to add image object to S3. " "key=%(obj_name)s") % { 'obj_name': obj_name }) raise glance.store.BackendException(msg)
def _do_request(self, method, url, body, headers): """ Connects to the server and issues a request. Handles converting any returned HTTP error status codes to OpenStack/Glance exceptions and closing the server connection. Returns the result data, or raises an appropriate exception. :param method: HTTP method ("GET", "POST", "PUT", etc...) :param url: urlparse.ParsedResult object with URL information :param body: data to send (as string, filelike or iterable), or None (default) :param headers: mapping of key/value pairs to add as headers :note If the body param has a read attribute, and method is either POST or PUT, this method will automatically conduct a chunked-transfer encoding and use the body as a file object or iterable, transferring chunks of data using the connection's send() method. This allows large objects to be transferred efficiently without buffering the entire body in memory. """ if url.query: path = url.path + "?" + url.query else: path = url.path try: connection_type = self.get_connection_type() headers = self._encode_headers(headers or {}) headers.update(osprofiler.web.get_trace_id_headers()) if 'x-auth-token' not in headers and self.auth_token: headers['x-auth-token'] = self.auth_token c = connection_type(url.hostname, url.port, **self.connect_kwargs) def _pushing(method): return method.lower() in ('post', 'put') def _simple(body): return body is None or isinstance(body, bytes) def _filelike(body): return hasattr(body, 'read') def _sendbody(connection, iter): connection.endheaders() for sent in iter: # iterator has done the heavy lifting pass def _chunkbody(connection, iter): connection.putheader('Transfer-Encoding', 'chunked') connection.endheaders() for chunk in iter: connection.send('%x\r\n%s\r\n' % (len(chunk), chunk)) connection.send('0\r\n\r\n') # Do a simple request or a chunked request, depending # on whether the body param is file-like or iterable and # the method is PUT or POST # if not _pushing(method) or _simple(body): # Simple request... c.request(method, path, body, headers) elif _filelike(body) or self._iterable(body): c.putrequest(method, path) use_sendfile = self._sendable(body) # According to HTTP/1.1, Content-Length and Transfer-Encoding # conflict. for header, value in headers.items(): if use_sendfile or header.lower() != 'content-length': c.putheader(header, str(value)) iter = utils.chunkreadable(body) if use_sendfile: # send actual file without copying into userspace _sendbody(c, iter) else: # otherwise iterate and chunk _chunkbody(c, iter) else: raise TypeError('Unsupported image type: %s' % body.__class__) res = c.getresponse() def _retry(res): return res.getheader('Retry-After') status_code = self.get_status_code(res) if status_code in self.OK_RESPONSE_CODES: return res elif status_code in self.REDIRECT_RESPONSE_CODES: raise exception.RedirectException(res.getheader('Location')) elif status_code == http_client.UNAUTHORIZED: raise exception.NotAuthenticated(res.read()) elif status_code == http_client.FORBIDDEN: raise exception.Forbidden(res.read()) elif status_code == http_client.NOT_FOUND: raise exception.NotFound(res.read()) elif status_code == http_client.CONFLICT: raise exception.Duplicate(res.read()) elif status_code == http_client.BAD_REQUEST: raise exception.Invalid(res.read()) elif status_code == http_client.MULTIPLE_CHOICES: raise exception.MultipleChoices(body=res.read()) elif status_code == http_client.REQUEST_ENTITY_TOO_LARGE: raise exception.LimitExceeded(retry=_retry(res), body=res.read()) elif status_code == http_client.INTERNAL_SERVER_ERROR: raise exception.ServerError() elif status_code == http_client.SERVICE_UNAVAILABLE: raise exception.ServiceUnavailable(retry=_retry(res)) else: raise exception.UnexpectedStatus(status=status_code, body=res.read()) except (socket.error, IOError) as e: raise exception.ClientConnectionError(e)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ checksum = hashlib.md5() image_name = str(image_id) with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn: fsid = None if hasattr(conn, 'get_fsid'): fsid = conn.get_fsid() with conn.open_ioctx(self.pool) as ioctx: order = int(math.log(self.chunk_size, 2)) LOG.debug('creating image %(name)s with order %(order)d and ' 'size %(size)d', {'name': text_type(image_name), 'order': order, 'size': image_size}) if image_size == 0: LOG.warning(_("since image size is zero we will be doing " "resize-before-write for each chunk which " "will be considerably slower than normal")) try: loc = self._create_image(fsid, ioctx, image_name, image_size, order) except rbd.ImageExists: raise exception.Duplicate( _('RBD image %s already exists') % image_id) try: with rbd.Image(ioctx, image_name) as image: bytes_written = 0 offset = 0 chunks = utils.chunkreadable(image_file, self.chunk_size) for chunk in chunks: # If the image size provided is zero we need to do # a resize for the amount we are writing. This will # be slower so setting a higher chunk size may # speed things up a bit. if image_size == 0: chunk_length = len(chunk) length = offset + chunk_length bytes_written += chunk_length LOG.debug("resizing image to %s KiB" % (length / units.Ki)) image.resize(length) LOG.debug("writing chunk at offset %s" % (offset)) offset += image.write(chunk, offset) checksum.update(chunk) if loc.snapshot: image.create_snap(loc.snapshot) image.protect_snap(loc.snapshot) except Exception: with excutils.save_and_reraise_exception(): # Delete image if one was created try: self._delete_image(loc.image, loc.snapshot) except exception.NotFound: pass # Make sure we send back the image size whether provided or inferred. if image_size == 0: image_size = bytes_written return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns an `glance.store.ImageAddResult` object containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval `glance.store.ImageAddResult` object :raises `glance.common.exception.Duplicate` if the image already existed S3 writes the image data using the scheme: s3://<ACCESS_KEY>:<SECRET_KEY>@<S3_URL>/<BUCKET>/<OBJ> where: <USER> = ``s3_store_user`` <KEY> = ``s3_store_key`` <S3_HOST> = ``s3_store_host`` <BUCKET> = ``s3_store_bucket`` <ID> = The id of the image being added """ from boto.s3.connection import S3Connection loc = StoreLocation({'scheme': self.scheme, 'bucket': self.bucket, 'key': image_id, 's3serviceurl': self.full_s3_host, 'accesskey': self.access_key, 'secretkey': self.secret_key}) s3_conn = S3Connection(loc.accesskey, loc.secretkey, host=loc.s3serviceurl, is_secure=(loc.scheme == 's3+https')) create_bucket_if_missing(self.bucket, s3_conn, self.conf) bucket_obj = get_bucket(s3_conn, self.bucket) obj_name = str(image_id) key = bucket_obj.get_key(obj_name) if key and key.exists(): raise exception.Duplicate(_("S3 already has an image at " "location %s") % loc.get_uri()) msg = _("Adding image object to S3 using (s3_host=%(s3_host)s, " "access_key=%(access_key)s, bucket=%(bucket)s, " "key=%(obj_name)s)") % ({'s3_host': self.s3_host, 'access_key': self.access_key, 'bucket': self.bucket, 'obj_name': obj_name}) logger.debug(msg) key = bucket_obj.new_key(obj_name) # We need to wrap image_file, which is a reference to the # webob.Request.body_file, with a seekable file-like object, # otherwise the call to set_contents_from_file() will die # with an error about Input object has no method 'seek'. We # might want to call webob.Request.make_body_seekable(), but # unfortunately, that method copies the entire image into # memory and results in LP Bug #818292 occurring. So, here # we write temporary file in as memory-efficient manner as # possible and then supply the temporary file to S3. We also # take this opportunity to calculate the image checksum while # writing the tempfile, so we don't need to call key.compute_md5() msg = _("Writing request body file to temporary file " "for %s") % loc.get_uri() logger.debug(msg) tmpdir = self.s3_store_object_buffer_dir temp_file = tempfile.NamedTemporaryFile(dir=tmpdir) checksum = hashlib.md5() for chunk in utils.chunkreadable(image_file, self.CHUNKSIZE): checksum.update(chunk) temp_file.write(chunk) temp_file.flush() msg = _("Uploading temporary file to S3 for %s") % loc.get_uri() logger.debug(msg) # OK, now upload the data into the key key.set_contents_from_file(open(temp_file.name, 'r+b'), replace=False) size = key.size checksum_hex = checksum.hexdigest() logger.debug(_("Wrote %(size)d bytes to S3 key named %(obj_name)s " "with checksum %(checksum_hex)s") % locals()) return (loc.get_uri(), size, checksum_hex)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns an `glance.store.ImageAddResult` object containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval `glance.store.ImageAddResult` object :raises `glance.common.exception.Duplicate` if the image already existed :note By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ full_data_path = self.path + "/" + image_id LOG.debug("connecting to %(host)s for %(data)s" % ({'host': self.host, 'data': full_data_path})) conn, err = rcConnect(self.host, self.port, self.user, self.zone) status = clientLoginWithPassword(conn, self.password) LOG.debug("attempting to open irods file '%s'" % full_data_path) f = iRodsOpen(conn, full_data_path, "w") if f is None: conn.close() raise exception.Duplicate(_("image file %s already exists " + "or no perms") % filepath) LOG.debug("performing the write") checksum = hashlib.md5() bytes_written = 0 try: for buf in utils.chunkreadable(image_file, ChunkedFile.CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) except Exception: # let's attempt an delete f.delete() reason = _('paritial write, transfer failed') LOG.error(reason) f.close() conn.disconnect() raise exception.StorageWriteDenied(reason) f.close() checksum_hex = checksum.hexdigest() LOG.debug("Wrote %(bytes)d bytes to %(path)s, " + "checksum = %(checksum)s" % ({'bytes': bytes_written, 'path': full_data_path, 'checksum': checksum_hex})) loc = StoreLocation({'scheme': self.scheme, 'host': self.host, 'port': self.port, 'zone': self.zone, 'path': self.path, 'user': self.user, 'password': self.password, 'data_name': image_id}) return (loc.get_uri(), bytes_written, checksum_hex)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns an `glance.store.ImageAddResult` object containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval `glance.store.ImageAddResult` object :raises `glance.common.exception.Duplicate` if the image already existed S3 writes the image data using the scheme: s3://<ACCESS_KEY>:<SECRET_KEY>@<S3_URL>/<BUCKET>/<OBJ> where: <USER> = ``s3_store_user`` <KEY> = ``s3_store_key`` <S3_HOST> = ``s3_store_host`` <BUCKET> = ``s3_store_bucket`` <ID> = The id of the image being added """ from boto.s3.connection import S3Connection loc = StoreLocation({ 'scheme': self.scheme, 'bucket': self.bucket, 'key': image_id, 's3serviceurl': self.full_s3_host, 'accesskey': self.access_key, 'secretkey': self.secret_key }) s3_conn = S3Connection(loc.accesskey, loc.secretkey, host=loc.s3serviceurl, is_secure=(loc.scheme == 's3+https'), calling_format=get_calling_format()) create_bucket_if_missing(self.bucket, s3_conn) bucket_obj = get_bucket(s3_conn, self.bucket) obj_name = str(image_id) def _sanitize(uri): return re.sub('//.*:.*@', '//s3_store_secret_key:s3_store_access_key@', uri) key = bucket_obj.get_key(obj_name) if key and key.exists(): raise exception.Duplicate( _("S3 already has an image at " "location %s") % _sanitize(loc.get_uri())) msg = _("Adding image object to S3 using (s3_host=%(s3_host)s, " "access_key=%(access_key)s, bucket=%(bucket)s, " "key=%(obj_name)s)") % ({ 's3_host': self.s3_host, 'access_key': self.access_key, 'bucket': self.bucket, 'obj_name': obj_name }) LOG.debug(msg) key = bucket_obj.new_key(obj_name) # We need to wrap image_file, which is a reference to the # webob.Request.body_file, with a seekable file-like object, # otherwise the call to set_contents_from_file() will die # with an error about Input object has no method 'seek'. We # might want to call webob.Request.make_body_seekable(), but # unfortunately, that method copies the entire image into # memory and results in LP Bug #818292 occurring. So, here # we write temporary file in as memory-efficient manner as # possible and then supply the temporary file to S3. We also # take this opportunity to calculate the image checksum while # writing the tempfile, so we don't need to call key.compute_md5() msg = _("Writing request body file to temporary file " "for %s") % _sanitize(loc.get_uri()) LOG.debug(msg) tmpdir = self.s3_store_object_buffer_dir temp_file = tempfile.NamedTemporaryFile(dir=tmpdir) checksum = hashlib.md5() for chunk in utils.chunkreadable(image_file, self.CHUNKSIZE): checksum.update(chunk) temp_file.write(chunk) temp_file.flush() msg = (_("Uploading temporary file to S3 for %s") % _sanitize(loc.get_uri())) LOG.debug(msg) # OK, now upload the data into the key key.set_contents_from_file(open(temp_file.name, 'r+b'), replace=False) size = key.size checksum_hex = checksum.hexdigest() LOG.debug( _("Wrote %(size)d bytes to S3 key named %(obj_name)s " "with checksum %(checksum_hex)s") % locals()) return (loc.get_uri(), size, checksum_hex)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed S3 writes the image data using the scheme: s3://<ACCESS_KEY>:<SECRET_KEY>@<S3_URL>/<BUCKET>/<OBJ> where: <USER> = ``s3_store_user`` <KEY> = ``s3_store_key`` <S3_HOST> = ``s3_store_host`` <BUCKET> = ``s3_store_bucket`` <ID> = The id of the image being added """ from boto.s3.connection import S3Connection loc = StoreLocation({'scheme': self.scheme, 'bucket': self.bucket, 'key': image_id, 's3serviceurl': self.full_s3_host, 'accesskey': self.access_key, 'secretkey': self.secret_key}) s3_conn = S3Connection(loc.accesskey, loc.secretkey, host=loc.s3serviceurl, is_secure=(loc.scheme == 's3+https'), calling_format=get_calling_format()) create_bucket_if_missing(self.bucket, s3_conn) bucket_obj = get_bucket(s3_conn, self.bucket) obj_name = str(image_id) def _sanitize(uri): return re.sub('//.*:.*@', '//s3_store_secret_key:s3_store_access_key@', uri) key = bucket_obj.get_key(obj_name) if key and key.exists(): raise exception.Duplicate(_("S3 already has an image at " "location %s") % _sanitize(loc.get_uri())) msg = ("Adding image object to S3 using (s3_host=%(s3_host)s, " "access_key=%(access_key)s, bucket=%(bucket)s, " "key=%(obj_name)s)" % ({'s3_host': self.s3_host, 'access_key': self.access_key, 'bucket': self.bucket, 'obj_name': obj_name})) LOG.debug(msg) LOG.debug("Uploading an image file to S3 for %s" % _sanitize(loc.get_uri())) if image_size < self.s3_store_large_object_size: key = bucket_obj.new_key(obj_name) # We need to wrap image_file, which is a reference to the # webob.Request.body_file, with a seekable file-like object, # otherwise the call to set_contents_from_file() will die # with an error about Input object has no method 'seek'. We # might want to call webob.Request.make_body_seekable(), but # unfortunately, that method copies the entire image into # memory and results in LP Bug #818292 occurring. So, here # we write temporary file in as memory-efficient manner as # possible and then supply the temporary file to S3. We also # take this opportunity to calculate the image checksum while # writing the tempfile, so we don't need to call key.compute_md5() msg = ("Writing request body file to temporary file " "for %s") % _sanitize(loc.get_uri()) LOG.debug(msg) tmpdir = self.s3_store_object_buffer_dir temp_file = tempfile.NamedTemporaryFile(dir=tmpdir) checksum = hashlib.md5() for chunk in utils.chunkreadable(image_file, self.CHUNKSIZE): checksum.update(chunk) temp_file.write(chunk) temp_file.flush() msg = ("Uploading temporary file to S3 " "for %s") % _sanitize(loc.get_uri()) LOG.debug(msg) # OK, now upload the data into the key key.set_contents_from_file(open(temp_file.name, 'rb'), replace=False) size = key.size checksum_hex = checksum.hexdigest() LOG.debug("Wrote %(size)d bytes to S3 key named %(obj_name)s " "with checksum %(checksum_hex)s" % {'size': size, 'obj_name': obj_name, 'checksum_hex': checksum_hex}) return (loc.get_uri(), size, checksum_hex, {}) else: checksum = hashlib.md5() parts = int(math.ceil(float(image_size) / float(self.s3_store_large_object_chunk_size))) threads = parts pool_size = CONF.s3_store_thread_pools pool = eventlet.greenpool.GreenPool(size=pool_size) mpu = bucket_obj.initiate_multipart_upload(obj_name) LOG.debug("Multipart initiate key=%(obj_name)s, " "UploadId=%(UploadId)s" % {'obj_name': obj_name, 'UploadId': mpu.id}) cstart = 0 plist = [] it = utils.chunkreadable(image_file, self.s3_store_large_object_chunk_size) for p in range(threads): chunk = next(it) clen = len(chunk) checksum.update(chunk) fp = six.BytesIO(chunk) fp.seek(0) part = UploadPart(mpu, fp, cstart + 1, clen) pool.spawn_n(run_upload, part) plist.append(part) cstart += 1 pedict = {} total_size = 0 pool.waitall() for part in plist: pedict.update(part.etag) total_size += part.size success = True for part in plist: if not part.success: success = False if success: # Complete xml = get_mpu_xml(pedict) bucket_obj.complete_multipart_upload(obj_name, mpu.id, xml) checksum_hex = checksum.hexdigest() LOG.info(_LI("Multipart complete key=%(obj_name)s " "UploadId=%(UploadId)s " "Wrote %(total_size)d bytes to S3 key" "named %(obj_name)s " "with checksum %(checksum_hex)s") % {'obj_name': obj_name, 'UploadId': mpu.id, 'total_size': total_size, 'obj_name': obj_name, 'checksum_hex': checksum_hex}) return (loc.get_uri(), total_size, checksum_hex, {}) else: # Abort bucket_obj.cancel_multipart_upload(obj_name, mpu.id) LOG.error(_LE("Some parts failed to upload to S3. " "Aborted the object key=%(obj_name)s") % {'obj_name': obj_name}) msg = (_("Failed to add image object to S3. " "key=%(obj_name)s") % {'obj_name': obj_name}) raise glance.store.BackendException(msg)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ checksum = hashlib.md5() image_name = str(image_id) with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn: fsid = None if hasattr(conn, 'get_fsid'): fsid = conn.get_fsid() with conn.open_ioctx(self.pool) as ioctx: order = int(math.log(self.chunk_size, 2)) LOG.debug(_('creating image %(name)s with order %(order)d and ' 'size %(size)d'), {'name': text_type(image_name), 'order': order, 'size': image_size}) if image_size == 0: LOG.warning(_("since image size is zero we will be doing " "resize-before-write for each chunk which " "will be considerably slower than normal")) try: loc = self._create_image(fsid, ioctx, image_name, image_size, order) except rbd.ImageExists: raise exception.Duplicate( _('RBD image %s already exists') % image_id) try: with rbd.Image(ioctx, image_name) as image: bytes_written = 0 offset = 0 chunks = utils.chunkreadable(image_file, self.chunk_size) for chunk in chunks: # If the image size provided is zero we need to do # a resize for the amount we are writing. This will # be slower so setting a higher chunk size may # speed things up a bit. if image_size == 0: chunk_length = len(chunk) length = offset + chunk_length bytes_written += chunk_length LOG.debug(_("resizing image to %s KiB") % (length / units.Ki)) image.resize(length) LOG.debug(_("writing chunk at offset %s") % (offset)) offset += image.write(chunk, offset) checksum.update(chunk) if loc.snapshot: image.create_snap(loc.snapshot) image.protect_snap(loc.snapshot) except Exception as exc: # Delete image if one was created try: self._delete_image(loc.image, loc.snapshot) except exception.NotFound: pass raise exc # Make sure we send back the image size whether provided or inferred. if image_size == 0: image_size = bytes_written return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ checksum = hashlib.md5() image_name = str(image_id) with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn: fsid = None if hasattr(conn, 'get_fsid'): fsid = conn.get_fsid() with conn.open_ioctx(self.pool) as ioctx: order = int(math.log(self.chunk_size, 2)) LOG.debug('creating image %s with order %d and size %d', image_name, order, image_size) if image_size == 0: LOG.warning(_("since image size is zero we will be doing " "resize-before-write for each chunk which " "will be considerably slower than normal")) try: loc = self._create_image(fsid, ioctx, image_name, image_size, order) except rbd.ImageExists: raise exception.Duplicate( _('RBD image %s already exists') % image_id) try: with rbd.Image(ioctx, image_name) as image: offset = 0 chunks = utils.chunkreadable(image_file, self.chunk_size) for chunk in chunks: # If the image size provided is zero we need to do # a resize for the amount we are writing. This will # be slower so setting a higher chunk size may # speed things up a bit. if image_size == 0: length = offset + len(chunk) LOG.debug(_("resizing image to %s KiB") % (length / 1024)) image.resize(length) LOG.debug(_("writing chunk at offset %s") % (offset)) offset += image.write(chunk, offset) checksum.update(chunk) if loc.snapshot: image.create_snap(loc.snapshot) image.protect_snap(loc.snapshot) except: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exception. with excutils.save_and_reraise_exception(): self._delete_image(loc.image, loc.snapshot) return (loc.get_uri(), image_size, checksum.hexdigest(), {})