コード例 #1
0
ファイル: sheepdog.py プロジェクト: brandon-adams/glance
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns a tuple containing information
        about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval tuple of URL in backing store, bytes written, and checksum
        :raises `glance.common.exception.Duplicate` if the image already
                existed
        """

        image = SheepdogImage(self.addr, self.port, image_id,
                              self.chunk_size)
        if image.exist():
            raise exception.Duplicate(_("Sheepdog image %s already exists")
                                      % image_id)

        location = StoreLocation({'image': image_id})
        checksum = hashlib.md5()

        image.create(image_size)

        total = left = image_size
        while left > 0:
            length = min(self.chunk_size, left)
            data = image_file.read(length)
            image.write(data, total - left, length)
            left -= length
            checksum.update(data)

        return (location.get_uri(), image_size, checksum.hexdigest())
コード例 #2
0
ファイル: swift.py プロジェクト: CiscoSystems/glance
    def get(self, location, connection=None):
        location = location.store_location
        if not connection:
            connection = self.get_connection(location)

        try:
            resp_headers, resp_body = connection.get_object(
                    container=location.container, obj=location.obj,
                    resp_chunk_size=self.CHUNKSIZE)
        except swiftclient.ClientException as e:
            if e.http_status == httplib.NOT_FOUND:
                uri = location.get_uri()
                msg = _("Swift could not find image at URI.")
                raise exception.NotFound(msg)
            else:
                raise

        class ResponseIndexable(glance.store.Indexable):
            def another(self):
                try:
                    return self.wrapped.next()
                except StopIteration:
                    return ''

        length = int(resp_headers.get('content-length', 0))
        return (ResponseIndexable(resp_body, length), length)
コード例 #3
0
ファイル: swift.py プロジェクト: CiscoSystems/glance
    def set_acls(self, location, public=False, read_tenants=None,
                 write_tenants=None, connection=None):
        location = location.store_location
        if not connection:
            connection = self.get_connection(location)

        if read_tenants is None:
            read_tenants = []
        if write_tenants is None:
            write_tenants = []

        headers = {}
        if public:
            headers['X-Container-Read'] = ".r:*"
        elif read_tenants:
            headers['X-Container-Read'] = ','.join(read_tenants)
        else:
            headers['X-Container-Read'] = ''

        write_tenants.extend(self.admin_tenants)
        if write_tenants:
            headers['X-Container-Write'] = ','.join(write_tenants)
        else:
            headers['X-Container-Write'] = ''

        try:
            connection.post_container(location.container, headers=headers)
        except swiftclient.ClientException as e:
            if e.http_status == httplib.NOT_FOUND:
                uri = location.get_uri()
                msg = _("Swift could not find image at URI.")
                raise exception.NotFound(msg)
            else:
                raise
コード例 #4
0
ファイル: swift.py プロジェクト: zhangheng1442/openstack
    def set_acls(self, location, public=False, read_tenants=None,
                 write_tenants=None, connection=None):
        location = location.store_location
        if not connection:
            connection = self.get_connection(location)

        if read_tenants is None:
            read_tenants = []
        if write_tenants is None:
            write_tenants = []

        headers = {}
        if public:
            headers['X-Container-Read'] = ".r:*"
        elif read_tenants:
            headers['X-Container-Read'] = ','.join(read_tenants)
        else:
            headers['X-Container-Read'] = ''

        write_tenants.extend(self.admin_tenants)
        if write_tenants:
            headers['X-Container-Write'] = ','.join(write_tenants)
        else:
            headers['X-Container-Write'] = ''

        try:
            connection.post_container(location.container, headers=headers)
        except swiftclient.ClientException, e:
            if e.http_status == httplib.NOT_FOUND:
                uri = location.get_uri()
                msg = _("Swift could not find image at URI.")
                raise exception.NotFound(msg)
            else:
                raise
コード例 #5
0
ファイル: rbd.py プロジェクト: nicoleLiu/glance
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns an `glance.store.ImageAddResult` object
        containing information about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval `glance.store.ImageAddResult` object
        :raises `glance.common.exception.Duplicate` if the image already
                existed
        """
        location = StoreLocation({"image": image_id})
        checksum = hashlib.md5()
        image_name = str(image_id)
        with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
            with conn.open_ioctx(self.pool) as ioctx:
                order = int(math.log(self.chunk_size, 2))
                logger.debug("creating image %s with order %d", image_name, order)
                try:
                    rbd.RBD().create(ioctx, image_name, image_size, order)
                except rbd.ImageExists:
                    raise exception.Duplicate(_("RBD image %s already exists") % image_id)
                with rbd.Image(ioctx, image_name) as image:
                    bytes_left = image_size
                    while bytes_left > 0:
                        length = min(self.chunk_size, bytes_left)
                        data = image_file.read(length)
                        image.write(data, image_size - bytes_left)
                        bytes_left -= length
                        checksum.update(data)

        return (location.get_uri(), image_size, checksum.hexdigest())
コード例 #6
0
    def get(self, location, connection=None):
        location = location.store_location
        if not connection:
            connection = self.get_connection(location)

        try:
            resp_headers, resp_body = connection.get_object(
                container=location.container,
                obj=location.obj,
                resp_chunk_size=self.CHUNKSIZE)
        except swiftclient.ClientException as e:
            if e.http_status == httplib.NOT_FOUND:
                uri = location.get_uri()
                msg = _("Swift could not find image at URI.")
                raise exception.NotFound(msg)
            else:
                raise

        class ResponseIndexable(glance.store.Indexable):
            def another(self):
                try:
                    return self.wrapped.next()
                except StopIteration:
                    return ''

        length = int(resp_headers.get('content-length', 0))
        return (ResponseIndexable(resp_body, length), length)
コード例 #7
0
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns a tuple containing information
        about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval tuple of URL in backing store, bytes written, and checksum
        :raises `glance.common.exception.Duplicate` if the image already
                existed
        """

        image = SheepdogImage(self.addr, self.port, image_id,
                              self.chunk_size)
        if image.exist():
            raise exception.Duplicate(_("Sheepdog image %s already exists")
                                      % image_id)

        location = StoreLocation({'image': image_id})
        checksum = hashlib.md5()

        image.create(image_size)

        total = left = image_size
        while left > 0:
            length = min(self.chunk_size, left)
            data = image_file.read(length)
            image.write(data, total - left, length)
            left -= length
            checksum.update(data)

        return (location.get_uri(), image_size, checksum.hexdigest(), {})
コード例 #8
0
ファイル: swift.py プロジェクト: zhangheng1442/openstack
    def get(self, location, connection=None):
        location = location.store_location
        if not connection:
            connection = self.get_connection(location)

        try:
            resp_headers, resp_body = connection.get_object(
                    container=location.container, obj=location.obj,
                    resp_chunk_size=self.CHUNKSIZE)
        except swiftclient.ClientException, e:
            if e.http_status == httplib.NOT_FOUND:
                uri = location.get_uri()
                msg = _("Swift could not find image at URI.")
                raise exception.NotFound(msg)
            else:
                raise
コード例 #9
0
ファイル: swift.py プロジェクト: hanlind/glance
    def get(self, location, connection=None):
        location = location.store_location
        if not connection:
            connection = self.get_connection(location)

        try:
            resp_headers, resp_body = connection.get_object(
                    container=location.container, obj=location.obj,
                    resp_chunk_size=self.CHUNKSIZE)
        except swiftclient.ClientException, e:
            if e.http_status == httplib.NOT_FOUND:
                uri = location.get_uri()
                raise exception.NotFound(_("Swift could not find image at "
                                           "uri %(uri)s") % locals())
            else:
                raise
コード例 #10
0
ファイル: rbd.py プロジェクト: CiscoSystems/glance
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns a tuple containing information
        about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval tuple of URL in backing store, bytes written, and checksum
        :raises `glance.common.exception.Duplicate` if the image already
                existed
        """
        checksum = hashlib.md5()
        image_name = str(image_id)
        with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
            fsid = None
            if hasattr(conn, 'get_fsid'):
                fsid = conn.get_fsid()
            with conn.open_ioctx(self.pool) as ioctx:
                order = int(math.log(self.chunk_size, 2))
                LOG.debug('creating image %s with order %d',
                          image_name, order)
                try:
                    location = self._create_image(fsid, ioctx, image_name,
                                                  image_size, order)
                except rbd.ImageExists:
                    raise exception.Duplicate(
                        _('RBD image %s already exists') % image_id)
                with rbd.Image(ioctx, image_name) as image:
                    bytes_left = image_size
                    while bytes_left > 0:
                        length = min(self.chunk_size, bytes_left)
                        data = image_file.read(length)
                        image.write(data, image_size - bytes_left)
                        bytes_left -= length
                        checksum.update(data)
                    if location.snapshot:
                        image.create_snap(location.snapshot)
                        image.protect_snap(location.snapshot)

        return (location.get_uri(), image_size, checksum.hexdigest())
コード例 #11
0
ファイル: rbd.py プロジェクト: bopopescu/Openstack-11
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns an `glance.store.ImageAddResult` object
        containing information about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval `glance.store.ImageAddResult` object
        :raises `glance.common.exception.Duplicate` if the image already
                existed
        """
        checksum = hashlib.md5()
        image_name = str(image_id)
        with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
            fsid = None
            if hasattr(conn, 'get_fsid'):
                fsid = conn.get_fsid()
            with conn.open_ioctx(self.pool) as ioctx:
                order = int(math.log(self.chunk_size, 2))
                LOG.debug('creating image %s with order %d', image_name, order)
                try:
                    location = self._create_image(fsid, ioctx, image_name,
                                                  image_size, order)
                except rbd.ImageExists:
                    raise exception.Duplicate(
                        _('RBD image %s already exists') % image_id)
                with rbd.Image(ioctx, image_name) as image:
                    bytes_left = image_size
                    while bytes_left > 0:
                        length = min(self.chunk_size, bytes_left)
                        data = image_file.read(length)
                        image.write(data, image_size - bytes_left)
                        bytes_left -= length
                        checksum.update(data)
                    if location.snapshot:
                        image.create_snap(location.snapshot)
                        image.protect_snap(location.snapshot)

        return (location.get_uri(), image_size, checksum.hexdigest())
コード例 #12
0
ファイル: swift.py プロジェクト: CiscoSystems/glance
    def delete(self, location, connection=None):
        location = location.store_location
        if not connection:
            connection = self.get_connection(location)

        try:
            # We request the manifest for the object. If one exists,
            # that means the object was uploaded in chunks/segments,
            # and we need to delete all the chunks as well as the
            # manifest.
            manifest = None
            try:
                headers = connection.head_object(
                        location.container, location.obj)
                manifest = headers.get('x-object-manifest')
            except swiftclient.ClientException as e:
                if e.http_status != httplib.NOT_FOUND:
                    raise
            if manifest:
                # Delete all the chunks before the object manifest itself
                obj_container, obj_prefix = manifest.split('/', 1)
                segments = connection.get_container(
                        obj_container, prefix=obj_prefix)[1]
                for segment in segments:
                    # TODO(jaypipes): This would be an easy area to parallelize
                    # since we're simply sending off parallelizable requests
                    # to Swift to delete stuff. It's not like we're going to
                    # be hogging up network or file I/O here...
                    connection.delete_object(obj_container,
                                             segment['name'])

            # Delete object (or, in segmented case, the manifest)
            connection.delete_object(location.container, location.obj)

        except swiftclient.ClientException as e:
            if e.http_status == httplib.NOT_FOUND:
                uri = location.get_uri()
                msg = _("Swift could not find image at URI.")
                raise exception.NotFound(msg)
            else:
                raise
コード例 #13
0
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns a tuple containing information
        about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval tuple of URL in backing store, bytes written, and checksum
        :raises `glance.common.exception.Duplicate` if the image already
                existed
        """

        image = SheepdogImage(self.addr, self.port, image_id,
                              self.WRITE_CHUNKSIZE)
        if image.exist():
            raise exception.Duplicate(
                _("Sheepdog image %s already exists") % image_id)

        location = StoreLocation({'image': image_id})
        checksum = hashlib.md5()

        image.create(image_size)

        try:
            total = left = image_size
            while left > 0:
                length = min(self.WRITE_CHUNKSIZE, left)
                data = image_file.read(length)
                image.write(data, total - left, length)
                left -= length
                checksum.update(data)
        except Exception:
            # Note(zhiyan): clean up already received data when
            # error occurs such as ImageSizeLimitExceeded exception.
            with excutils.save_and_reraise_exception():
                image.delete()

        return (location.get_uri(), image_size, checksum.hexdigest(), {})
コード例 #14
0
ファイル: rbd.py プロジェクト: yuyonglucky/glance
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns a tuple containing information
        about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval tuple of URL in backing store, bytes written, checksum
                and a dictionary with storage system specific information
        :raises `glance.common.exception.Duplicate` if the image already
                existed
        """
        checksum = hashlib.md5()
        image_name = str(image_id)
        with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
            fsid = None
            if hasattr(conn, 'get_fsid'):
                fsid = conn.get_fsid()
            with conn.open_ioctx(self.pool) as ioctx:
                order = int(math.log(self.chunk_size, 2))
                LOG.debug('creating image %s with order %d', image_name, order)
                try:
                    location = self._create_image(fsid, ioctx, image_name,
                                                  image_size, order)
                except rbd.ImageExists:
                    raise exception.Duplicate(
                        _('RBD image %s already exists') % image_id)
                with rbd.Image(ioctx, image_name) as image:
                    offset = 0
                    chunks = utils.chunkreadable(image_file, self.chunk_size)
                    for chunk in chunks:
                        offset += image.write(chunk, offset)
                        checksum.update(chunk)
                    if location.snapshot:
                        image.create_snap(location.snapshot)
                        image.protect_snap(location.snapshot)

        return (location.get_uri(), image_size, checksum.hexdigest(), {})
コード例 #15
0
ファイル: sheepdog.py プロジェクト: jokke-ilujo/glance.store
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns a tuple containing information
        about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval tuple of URL in backing store, bytes written, and checksum
        :raises `glance.store.exceptions.Duplicate` if the image already
                existed
        """

        image = SheepdogImage(self.addr, self.port, image_id,
                              self.chunk_size)
        if image.exist():
            raise exceptions.Duplicate(_("Sheepdog image %s already exists")
                                      % image_id)

        location = StoreLocation({'image': image_id})
        checksum = hashlib.md5()

        image.create(image_size)

        try:
            total = left = image_size
            while left > 0:
                length = min(self.chunk_size, left)
                data = image_file.read(length)
                image.write(data, total - left, length)
                left -= length
                checksum.update(data)
        except Exception:
            # Note(zhiyan): clean up already received data when
            # error occurs such as ImageSizeLimitExceeded exceptions.
            with excutils.save_and_reraise_exception():
                image.delete()

        return (location.get_uri(), image_size, checksum.hexdigest(), {})
コード例 #16
0
    def delete(self, location, connection=None):
        location = location.store_location
        if not connection:
            connection = self.get_connection(location)

        try:
            # We request the manifest for the object. If one exists,
            # that means the object was uploaded in chunks/segments,
            # and we need to delete all the chunks as well as the
            # manifest.
            manifest = None
            try:
                headers = connection.head_object(location.container,
                                                 location.obj)
                manifest = headers.get('x-object-manifest')
            except swiftclient.ClientException as e:
                if e.http_status != httplib.NOT_FOUND:
                    raise
            if manifest:
                # Delete all the chunks before the object manifest itself
                obj_container, obj_prefix = manifest.split('/', 1)
                segments = connection.get_container(obj_container,
                                                    prefix=obj_prefix)[1]
                for segment in segments:
                    # TODO(jaypipes): This would be an easy area to parallelize
                    # since we're simply sending off parallelizable requests
                    # to Swift to delete stuff. It's not like we're going to
                    # be hogging up network or file I/O here...
                    connection.delete_object(obj_container, segment['name'])

            # Delete object (or, in segmented case, the manifest)
            connection.delete_object(location.container, location.obj)

        except swiftclient.ClientException as e:
            if e.http_status == httplib.NOT_FOUND:
                uri = location.get_uri()
                msg = _("Swift could not find image at URI.")
                raise exception.NotFound(msg)
            else:
                raise
コード例 #17
0
ファイル: swift.py プロジェクト: vmcloud/glance
    def add(self, image_id, image_file, image_size, connection=None):
        location = self.create_location(image_id)
        if not connection:
            connection = self.get_connection(location)

        self._create_container_if_missing(location.container, connection)

        LOG.debug(_("Adding image object '%(obj_name)s' "
                    "to Swift") % dict(obj_name=location.obj))
        try:
            if image_size > 0 and image_size < self.large_object_size:
                # Image size is known, and is less than large_object_size.
                # Send to Swift with regular PUT.
                obj_etag = connection.put_object(location.container,
                                                 location.obj, image_file,
                                                 content_length=image_size)
            else:
                # Write the image into Swift in chunks.
                chunk_id = 1
                if image_size > 0:
                    total_chunks = str(int(
                        math.ceil(float(image_size) /
                                  float(self.large_object_chunk_size))))
                else:
                    # image_size == 0 is when we don't know the size
                    # of the image. This can occur with older clients
                    # that don't inspect the payload size.
                    LOG.debug(_("Cannot determine image size. Adding as a "
                                "segmented object to Swift."))
                    total_chunks = '?'

                checksum = hashlib.md5()
                written_chunks = []
                combined_chunks_size = 0
                while True:
                    chunk_size = self.large_object_chunk_size
                    if image_size == 0:
                        content_length = None
                    else:
                        left = image_size - combined_chunks_size
                        if left == 0:
                            break
                        if chunk_size > left:
                            chunk_size = left
                        content_length = chunk_size

                    chunk_name = "%s-%05d" % (location.obj, chunk_id)
                    reader = ChunkReader(image_file, checksum, chunk_size)
                    try:
                        chunk_etag = connection.put_object(
                            location.container, chunk_name, reader,
                            content_length=content_length)
                        written_chunks.append(chunk_name)
                    except Exception:
                        # Delete orphaned segments from swift backend
                        with excutils.save_and_reraise_exception():
                            LOG.exception(_("Error during chunked upload to "
                                            "backend, deleting stale chunks"))
                            self._delete_stale_chunks(connection,
                                                      location.container,
                                                      written_chunks)

                    bytes_read = reader.bytes_read
                    msg = (_("Wrote chunk %(chunk_name)s (%(chunk_id)d/"
                             "%(total_chunks)s) of length %(bytes_read)d "
                             "to Swift returning MD5 of content: "
                             "%(chunk_etag)s") %
                           {'chunk_name': chunk_name,
                            'chunk_id': chunk_id,
                            'total_chunks': total_chunks,
                            'bytes_read': bytes_read,
                            'chunk_etag': chunk_etag})
                    LOG.debug(msg)

                    if bytes_read == 0:
                        # Delete the last chunk, because it's of zero size.
                        # This will happen if size == 0.
                        LOG.debug(_("Deleting final zero-length chunk"))
                        connection.delete_object(location.container,
                                                 chunk_name)
                        break

                    chunk_id += 1
                    combined_chunks_size += bytes_read

                # In the case we have been given an unknown image size,
                # set the size to the total size of the combined chunks.
                if image_size == 0:
                    image_size = combined_chunks_size

                # Now we write the object manifest and return the
                # manifest's etag...
                manifest = "%s/%s-" % (location.container, location.obj)
                headers = {'ETag': hashlib.md5("").hexdigest(),
                           'X-Object-Manifest': manifest}

                # The ETag returned for the manifest is actually the
                # MD5 hash of the concatenated checksums of the strings
                # of each chunk...so we ignore this result in favour of
                # the MD5 of the entire image file contents, so that
                # users can verify the image file contents accordingly
                connection.put_object(location.container, location.obj,
                                      None, headers=headers)
                obj_etag = checksum.hexdigest()

            # NOTE: We return the user and key here! Have to because
            # location is used by the API server to return the actual
            # image data. We *really* should consider NOT returning
            # the location attribute from GET /images/<ID> and
            # GET /images/details

            return (location.get_uri(), image_size, obj_etag, {})
        except swiftclient.ClientException as e:
            if e.http_status == httplib.CONFLICT:
                raise exception.Duplicate(_("Swift already has an image at "
                                            "this location"))
            msg = (_("Failed to add object to Swift.\n"
                     "Got error from Swift: %(e)s") % {'e': e})
            LOG.error(msg)
            raise glance.store.BackendException(msg)
コード例 #18
0
ファイル: swift.py プロジェクト: Cerberus98/glance
    def add(self, image_id, image_file):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns an `glance.store.ImageAddResult` object
        containing information about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object

        :retval `glance.store.ImageAddResult` object
        :raises `glance.common.exception.Duplicate` if the image already
                existed

        Swift writes the image data using the scheme:
            ``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
        where:
            <USER> = ``swift_store_user``
            <KEY> = ``swift_store_key``
            <AUTH_ADDRESS> = ``swift_store_auth_address``
            <CONTAINER> = ``swift_store_container``
            <ID> = The id of the image being added

        :note Swift auth URLs by default use HTTPS. To specify an HTTP
              auth URL, you can specify http://someurl.com for the
              swift_store_auth_address config option
        """
        swift_conn = self._make_swift_connection(
            auth_url=self.full_auth_address, user=self.user, key=self.key)

        create_container_if_missing(self.container, swift_conn, self.options)

        obj_name = str(image_id)
        location = StoreLocation({'scheme': self.scheme,
                                  'container': self.container,
                                  'obj': obj_name,
                                  'authurl': self.auth_address,
                                  'user': self.user,
                                  'key': self.key})

        logger.debug(_("Adding image object '%(obj_name)s' "
                       "to Swift") % locals())
        try:
            obj_etag = swift_conn.put_object(self.container, obj_name,
                                             image_file)

            # NOTE: We return the user and key here! Have to because
            # location is used by the API server to return the actual
            # image data. We *really* should consider NOT returning
            # the location attribute from GET /images/<ID> and
            # GET /images/details

            # We do a HEAD on the newly-added image to determine the size
            # of the image. A bit slow, but better than taking the word
            # of the user adding the image with size attribute in the metadata
            resp_headers = swift_conn.head_object(self.container, obj_name)
            size = 0
            # header keys are lowercased by Swift
            if 'content-length' in resp_headers:
                size = int(resp_headers['content-length'])
            return (location.get_uri(), size, obj_etag)
        except swift_client.ClientException, e:
            if e.http_status == httplib.CONFLICT:
                raise exception.Duplicate(_("Swift already has an image at "
                                          "location %s") % location.get_uri())
            msg = (_("Failed to add object to Swift.\n"
                   "Got error from Swift: %(e)s") % locals())
            logger.error(msg)
            raise glance.store.BackendException(msg)
コード例 #19
0
ファイル: swift.py プロジェクト: stateman/glance
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns an `glance.store.ImageAddResult` object
        containing information about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval `glance.store.ImageAddResult` object
        :raises `glance.common.exception.Duplicate` if the image already
                existed

        Swift writes the image data using the scheme:
            ``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
        where:
            <USER> = ``swift_store_user``
            <KEY> = ``swift_store_key``
            <AUTH_ADDRESS> = ``swift_store_auth_address``
            <CONTAINER> = ``swift_store_container``
            <ID> = The id of the image being added

        :note Swift auth URLs by default use HTTPS. To specify an HTTP
              auth URL, you can specify http://someurl.com for the
              swift_store_auth_address config option

        :note Swift cannot natively/transparently handle objects >5GB
              in size. So, if the image is greater than 5GB, we write
              chunks of image data to Swift and then write an manifest
              to Swift that contains information about the chunks.
              This same chunking process is used by default for images
              of an unknown size, as pushing them directly to swift would
              fail if the image turns out to be greater than 5GB.
        """
        swift_conn = self._make_swift_connection(self.full_auth_address,
                                                 self.user,
                                                 self.key,
                                                 storage_url=self.storage_url,
                                                 token=self.token)

        obj_name = str(image_id)
        if self.multi_tenant:
            # NOTE: When using multi-tenant we create containers for each
            # image so we can set permissions on each image in swift
            container = self.container + '_' + obj_name
            auth_or_store_url = self.storage_url
        else:
            container = self.container
            auth_or_store_url = self.auth_address

        create_container_if_missing(container, swift_conn)

        location = StoreLocation({
            'scheme': self.scheme,
            'container': container,
            'obj': obj_name,
            'auth_or_store_url': auth_or_store_url,
            'user': self.user,
            'key': self.key
        })

        LOG.debug(
            _("Adding image object '%(obj_name)s' "
              "to Swift") % locals())
        try:
            if image_size > 0 and image_size < self.large_object_size:
                # Image size is known, and is less than large_object_size.
                # Send to Swift with regular PUT.
                obj_etag = swift_conn.put_object(container,
                                                 obj_name,
                                                 image_file,
                                                 content_length=image_size)
            else:
                # Write the image into Swift in chunks.
                chunk_id = 1
                if image_size > 0:
                    total_chunks = str(
                        int(
                            math.ceil(
                                float(image_size) /
                                float(self.large_object_chunk_size))))
                else:
                    # image_size == 0 is when we don't know the size
                    # of the image. This can occur with older clients
                    # that don't inspect the payload size.
                    LOG.debug(
                        _("Cannot determine image size. Adding as a "
                          "segmented object to Swift."))
                    total_chunks = '?'

                checksum = hashlib.md5()
                combined_chunks_size = 0
                while True:
                    chunk_size = self.large_object_chunk_size
                    if image_size == 0:
                        content_length = None
                    else:
                        left = image_size - combined_chunks_size
                        if left == 0:
                            break
                        if chunk_size > left:
                            chunk_size = left
                        content_length = chunk_size

                    chunk_name = "%s-%05d" % (obj_name, chunk_id)
                    reader = ChunkReader(image_file, checksum, chunk_size)
                    chunk_etag = swift_conn.put_object(
                        container,
                        chunk_name,
                        reader,
                        content_length=content_length)
                    bytes_read = reader.bytes_read
                    msg = _("Wrote chunk %(chunk_name)s (%(chunk_id)d/"
                            "%(total_chunks)s) of length %(bytes_read)d "
                            "to Swift returning MD5 of content: "
                            "%(chunk_etag)s")
                    LOG.debug(msg % locals())

                    if bytes_read == 0:
                        # Delete the last chunk, because it's of zero size.
                        # This will happen if image_size == 0.
                        LOG.debug(_("Deleting final zero-length chunk"))
                        swift_conn.delete_object(container, chunk_name)
                        break

                    chunk_id += 1
                    combined_chunks_size += bytes_read

                # In the case we have been given an unknown image size,
                # set the image_size to the total size of the combined chunks.
                if image_size == 0:
                    image_size = combined_chunks_size

                # Now we write the object manifest and return the
                # manifest's etag...
                manifest = "%s/%s" % (container, obj_name)
                headers = {
                    'ETag': hashlib.md5("").hexdigest(),
                    'X-Object-Manifest': manifest
                }

                # The ETag returned for the manifest is actually the
                # MD5 hash of the concatenated checksums of the strings
                # of each chunk...so we ignore this result in favour of
                # the MD5 of the entire image file contents, so that
                # users can verify the image file contents accordingly
                swift_conn.put_object(container,
                                      obj_name,
                                      None,
                                      headers=headers)
                obj_etag = checksum.hexdigest()

            # NOTE: We return the user and key here! Have to because
            # location is used by the API server to return the actual
            # image data. We *really* should consider NOT returning
            # the location attribute from GET /images/<ID> and
            # GET /images/details

            return (location.get_uri(), image_size, obj_etag)
        except swiftclient.ClientException, e:
            if e.http_status == httplib.CONFLICT:
                raise exception.Duplicate(
                    _("Swift already has an image at "
                      "location %s") % location.get_uri())
            msg = (_("Failed to add object to Swift.\n"
                     "Got error from Swift: %(e)s") % locals())
            LOG.error(msg)
            raise glance.store.BackendException(msg)
コード例 #20
0
ファイル: swift.py プロジェクト: openstack-ja/glance
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns an `glance.store.ImageAddResult` object
        containing information about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval `glance.store.ImageAddResult` object
        :raises `glance.common.exception.Duplicate` if the image already
                existed

        Swift writes the image data using the scheme:
            ``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
        where:
            <USER> = ``swift_store_user``
            <KEY> = ``swift_store_key``
            <AUTH_ADDRESS> = ``swift_store_auth_address``
            <CONTAINER> = ``swift_store_container``
            <ID> = The id of the image being added

        :note Swift auth URLs by default use HTTPS. To specify an HTTP
              auth URL, you can specify http://someurl.com for the
              swift_store_auth_address config option

        :note Swift cannot natively/transparently handle objects >5GB
              in size. So, if the image is greater than 5GB, we write
              chunks of image data to Swift and then write an manifest
              to Swift that contains information about the chunks.
              This same chunking process is used by default for images
              of an unknown size, as pushing them directly to swift would
              fail if the image turns out to be greater than 5GB.
        """
        swift_conn = self._make_swift_connection(
            auth_url=self.full_auth_address, user=self.user, key=self.key)

        create_container_if_missing(self.container, swift_conn, self.conf)

        obj_name = str(image_id)
        location = StoreLocation({'scheme': self.scheme,
                                  'container': self.container,
                                  'obj': obj_name,
                                  'authurl': self.auth_address,
                                  'user': self.user,
                                  'key': self.key})

        logger.debug(_("Adding image object '%(obj_name)s' "
                       "to Swift") % locals())
        try:
            if image_size > 0 and image_size < self.large_object_size:
                # Image size is known, and is less than large_object_size.
                # Send to Swift with regular PUT.
                obj_etag = swift_conn.put_object(self.container, obj_name,
                                                 image_file,
                                                 content_length=image_size)
            else:
                # Write the image into Swift in chunks.
                chunk_id = 1
                if image_size > 0:
                    total_chunks = str(int(
                        math.ceil(float(image_size) /
                                  float(self.large_object_chunk_size))))
                else:
                    # image_size == 0 is when we don't know the size
                    # of the image. This can occur with older clients
                    # that don't inspect the payload size.
                    logger.debug(_("Cannot determine image size. Adding as a "
                                   "segmented object to Swift."))
                    total_chunks = '?'

                checksum = hashlib.md5()
                combined_chunks_size = 0
                while True:
                    chunk_size = self.large_object_chunk_size
                    if image_size == 0:
                        content_length = None
                    else:
                        left = image_size - combined_chunks_size
                        if left == 0:
                            break
                        if chunk_size > left:
                            chunk_size = left
                        content_length = chunk_size

                    chunk_name = "%s-%05d" % (obj_name, chunk_id)
                    reader = ChunkReader(image_file, checksum, chunk_size)
                    chunk_etag = swift_conn.put_object(
                        self.container, chunk_name, reader,
                        content_length=content_length)
                    bytes_read = reader.bytes_read
                    logger.debug(_("Wrote chunk %(chunk_id)d/"
                                   "%(total_chunks)s of length %(bytes_read)d "
                                   "to Swift returning MD5 of content: "
                                   "%(chunk_etag)s")
                                 % locals())

                    if bytes_read == 0:
                        # Delete the last chunk, because it's of zero size.
                        # This will happen if image_size == 0.
                        logger.debug(_("Deleting final zero-length chunk"))
                        swift_conn.delete_object(self.container, chunk_name)
                        break

                    chunk_id += 1
                    combined_chunks_size += bytes_read

                # In the case we have been given an unknown image size,
                # set the image_size to the total size of the combined chunks.
                if image_size == 0:
                    image_size = combined_chunks_size

                # Now we write the object manifest and return the
                # manifest's etag...
                manifest = "%s/%s" % (self.container, obj_name)
                headers = {'ETag': hashlib.md5("").hexdigest(),
                           'X-Object-Manifest': manifest}

                # The ETag returned for the manifest is actually the
                # MD5 hash of the concatenated checksums of the strings
                # of each chunk...so we ignore this result in favour of
                # the MD5 of the entire image file contents, so that
                # users can verify the image file contents accordingly
                swift_conn.put_object(self.container, obj_name,
                                      None, headers=headers)
                obj_etag = checksum.hexdigest()

            # NOTE: We return the user and key here! Have to because
            # location is used by the API server to return the actual
            # image data. We *really* should consider NOT returning
            # the location attribute from GET /images/<ID> and
            # GET /images/details

            return (location.get_uri(), image_size, obj_etag)
        except swift_client.ClientException, e:
            if e.http_status == httplib.CONFLICT:
                raise exception.Duplicate(_("Swift already has an image at "
                                          "location %s") % location.get_uri())
            msg = (_("Failed to add object to Swift.\n"
                   "Got error from Swift: %(e)s") % locals())
            logger.error(msg)
            raise glance.store.BackendException(msg)
コード例 #21
0
    def add(self, image_id, image_file, image_size, connection=None):
        location = self.create_location(image_id)
        if not connection:
            connection = self.get_connection(location)

        self._create_container_if_missing(location.container, connection)

        LOG.debug(
            _("Adding image object '%(obj_name)s' "
              "to Swift") % dict(obj_name=location.obj))
        try:
            if image_size > 0 and image_size < self.large_object_size:
                # Image size is known, and is less than large_object_size.
                # Send to Swift with regular PUT.
                obj_etag = connection.put_object(location.container,
                                                 location.obj,
                                                 image_file,
                                                 content_length=image_size)
            else:
                # Write the image into Swift in chunks.
                chunk_id = 1
                if image_size > 0:
                    total_chunks = str(
                        int(
                            math.ceil(
                                float(image_size) /
                                float(self.large_object_chunk_size))))
                else:
                    # image_size == 0 is when we don't know the size
                    # of the image. This can occur with older clients
                    # that don't inspect the payload size.
                    LOG.debug(
                        _("Cannot determine image size. Adding as a "
                          "segmented object to Swift."))
                    total_chunks = '?'

                checksum = hashlib.md5()
                written_chunks = []
                combined_chunks_size = 0
                while True:
                    chunk_size = self.large_object_chunk_size
                    if image_size == 0:
                        content_length = None
                    else:
                        left = image_size - combined_chunks_size
                        if left == 0:
                            break
                        if chunk_size > left:
                            chunk_size = left
                        content_length = chunk_size

                    chunk_name = "%s-%05d" % (location.obj, chunk_id)
                    reader = ChunkReader(image_file, checksum, chunk_size)
                    try:
                        chunk_etag = connection.put_object(
                            location.container,
                            chunk_name,
                            reader,
                            content_length=content_length)
                        written_chunks.append(chunk_name)
                    except Exception:
                        # Delete orphaned segments from swift backend
                        self._delete_stale_chunks(connection,
                                                  location.container,
                                                  written_chunks)
                        raise

                    bytes_read = reader.bytes_read
                    msg = _("Wrote chunk %(chunk_name)s (%(chunk_id)d/"
                            "%(total_chunks)s) of length %(bytes_read)d "
                            "to Swift returning MD5 of content: "
                            "%(chunk_etag)s")
                    LOG.debug(msg % locals())

                    if bytes_read == 0:
                        # Delete the last chunk, because it's of zero size.
                        # This will happen if size == 0.
                        LOG.debug(_("Deleting final zero-length chunk"))
                        connection.delete_object(location.container,
                                                 chunk_name)
                        break

                    chunk_id += 1
                    combined_chunks_size += bytes_read

                # In the case we have been given an unknown image size,
                # set the size to the total size of the combined chunks.
                if image_size == 0:
                    image_size = combined_chunks_size

                # Now we write the object manifest and return the
                # manifest's etag...
                manifest = "%s/%s-" % (location.container, location.obj)
                headers = {
                    'ETag': hashlib.md5("").hexdigest(),
                    'X-Object-Manifest': manifest
                }

                # The ETag returned for the manifest is actually the
                # MD5 hash of the concatenated checksums of the strings
                # of each chunk...so we ignore this result in favour of
                # the MD5 of the entire image file contents, so that
                # users can verify the image file contents accordingly
                connection.put_object(location.container,
                                      location.obj,
                                      None,
                                      headers=headers)
                obj_etag = checksum.hexdigest()

            # NOTE: We return the user and key here! Have to because
            # location is used by the API server to return the actual
            # image data. We *really* should consider NOT returning
            # the location attribute from GET /images/<ID> and
            # GET /images/details

            return (location.get_uri(), image_size, obj_etag)
        except swiftclient.ClientException as e:
            if e.http_status == httplib.CONFLICT:
                raise exception.Duplicate(
                    _("Swift already has an image at "
                      "this location"))
            msg = (_("Failed to add object to Swift.\n"
                     "Got error from Swift: %(e)s") % locals())
            LOG.error(msg)
            raise glance.store.BackendException(msg)
コード例 #22
0
ファイル: swift.py プロジェクト: emonty/glance
    def add(cls, id, data, options):
        """
        Stores image data to Swift and returns a location that the image was
        written to.

        Swift writes the image data using the scheme:
            ``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
        where:
            <USER> = ``swift_store_user``
            <KEY> = ``swift_store_key``
            <AUTH_ADDRESS> = ``swift_store_auth_address``
            <CONTAINER> = ``swift_store_container``
            <ID> = The id of the image being added

        :note Swift auth URLs by default use HTTPS. To specify an HTTP
              auth URL, you can specify http://someurl.com for the
              swift_store_auth_address config option

        :param id: The opaque image identifier
        :param data: The image data to write, as a file-like object
        :param options: Conf mapping

        :retval Tuple with (location, size)
                The location that was written,
                and the size in bytes of the data written
        """
        from swift.common import client as swift_client
        container = options.get('swift_store_container',
                                DEFAULT_SWIFT_CONTAINER)

        # TODO(jaypipes): This needs to be checked every time
        # because of the decision to make glance.store.Backend's
        # interface all @classmethods. This is inefficient. Backend
        # should be a stateful object with options parsed once in
        # a constructor.
        auth_address = cls._option_get(options, 'swift_store_auth_address')
        user = cls._option_get(options, 'swift_store_user')
        key = cls._option_get(options, 'swift_store_key')

        scheme = 'swift+https'
        if auth_address.startswith('http://'):
            scheme = 'swift+http'
            full_auth_address = auth_address
        elif auth_address.startswith('https://'):
            full_auth_address = auth_address
        else:
            full_auth_address = 'https://' + auth_address  # Defaults https

        swift_conn = swift_client.Connection(
            authurl=full_auth_address, user=user, key=key, snet=False)

        logger.debug("Adding image object to Swift using "
                     "(auth_address=%(auth_address)s, user=%(user)s, "
                     "key=%(key)s)" % locals())

        create_container_if_missing(container, swift_conn, options)

        obj_name = str(id)
        location = StoreLocation({'scheme': scheme,
                                  'container': container,
                                  'obj': obj_name,
                                  'authurl': auth_address,
                                  'user': user,
                                  'key': key})

        try:
            obj_etag = swift_conn.put_object(container, obj_name, data)

            # NOTE: We return the user and key here! Have to because
            # location is used by the API server to return the actual
            # image data. We *really* should consider NOT returning
            # the location attribute from GET /images/<ID> and
            # GET /images/details

            # We do a HEAD on the newly-added image to determine the size
            # of the image. A bit slow, but better than taking the word
            # of the user adding the image with size attribute in the metadata
            resp_headers = swift_conn.head_object(container, obj_name)
            size = 0
            # header keys are lowercased by Swift
            if 'content-length' in resp_headers:
                size = int(resp_headers['content-length'])
            return (location.get_uri(), size, obj_etag)
        except swift_client.ClientException, e:
            if e.http_status == httplib.CONFLICT:
                raise exception.Duplicate("Swift already has an image at "
                                          "location %(location)s" % locals())
            msg = ("Failed to add object to Swift.\n"
                   "Got error from Swift: %(e)s" % locals())
            raise glance.store.BackendException(msg)
コード例 #23
0
ファイル: swift.py プロジェクト: zhangheng1442/openstack
                segments = connection.get_container(
                        obj_container, prefix=obj_prefix)[1]
                for segment in segments:
                    # TODO(jaypipes): This would be an easy area to parallelize
                    # since we're simply sending off parallelizable requests
                    # to Swift to delete stuff. It's not like we're going to
                    # be hogging up network or file I/O here...
                    connection.delete_object(obj_container,
                                             segment['name'])

            # Delete object (or, in segmented case, the manifest)
            connection.delete_object(location.container, location.obj)

        except swiftclient.ClientException, e:
            if e.http_status == httplib.NOT_FOUND:
                uri = location.get_uri()
                msg = _("Swift could not find image at URI.")
                raise exception.NotFound(msg)
            else:
                raise

    def _create_container_if_missing(self, container, connection):
        """
        Creates a missing container in Swift if the
        ``swift_store_create_container_on_put`` option is set.

        :param container: Name of container to create
        :param connection: Connection to swift service
        """
        try:
            connection.head_container(container)
コード例 #24
0
ファイル: swift.py プロジェクト: athanatos/glance
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns an `glance.store.ImageAddResult` object
        containing information about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval `glance.store.ImageAddResult` object
        :raises `glance.common.exception.Duplicate` if the image already
                existed

        Swift writes the image data using the scheme:
            ``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
        where:
            <USER> = ``swift_store_user``
            <KEY> = ``swift_store_key``
            <AUTH_ADDRESS> = ``swift_store_auth_address``
            <CONTAINER> = ``swift_store_container``
            <ID> = The id of the image being added

        :note Swift auth URLs by default use HTTPS. To specify an HTTP
              auth URL, you can specify http://someurl.com for the
              swift_store_auth_address config option

        :note Swift cannot natively/transparently handle objects >5GB
              in size. So, if the image is greater than 5GB, we write
              chunks of image data to Swift and then write an manifest
              to Swift that contains information about the chunks.
        """
        swift_conn = self._make_swift_connection(
            auth_url=self.full_auth_address, user=self.user, key=self.key)

        create_container_if_missing(self.container, swift_conn, self.options)

        obj_name = str(image_id)
        location = StoreLocation({'scheme': self.scheme,
                                  'container': self.container,
                                  'obj': obj_name,
                                  'authurl': self.auth_address,
                                  'user': self.user,
                                  'key': self.key})

        logger.debug(_("Adding image object '%(obj_name)s' "
                       "to Swift") % locals())
        try:
            if image_size < self.large_object_size:
                # image_size == 0 is when we don't know the size
                # of the image. This can occur with older clients
                # that don't inspect the payload size, and we simply
                # try to put the object into Swift and hope for the
                # best...
                obj_etag = swift_conn.put_object(self.container, obj_name,
                                                 image_file)
            else:
                # Write the image into Swift in chunks. We cannot
                # stream chunks of the webob.Request.body_file, unfortunately,
                # so we must write chunks of the body_file into a temporary
                # disk buffer, and then pass this disk buffer to Swift.
                bytes_left = image_size
                chunk_id = 1
                total_chunks = int(math.ceil(
                    float(image_size) / float(self.large_object_chunk_size)))
                checksum = hashlib.md5()
                while bytes_left > 0:
                    with tempfile.NamedTemporaryFile() as disk_buffer:
                        chunk_size = min(self.large_object_chunk_size,
                                         bytes_left)
                        logger.debug(_("Writing %(chunk_size)d bytes for "
                                       "chunk %(chunk_id)d/"
                                       "%(total_chunks)d to disk buffer "
                                       "for image %(image_id)s")
                                     % locals())
                        chunk = image_file.read(chunk_size)
                        checksum.update(chunk)
                        disk_buffer.write(chunk)
                        disk_buffer.flush()
                        logger.debug(_("Writing chunk %(chunk_id)d/"
                                       "%(total_chunks)d to Swift "
                                       "for image %(image_id)s")
                                     % locals())
                        chunk_etag = swift_conn.put_object(
                            self.container,
                            "%s-%05d" % (obj_name, chunk_id),
                            open(disk_buffer.name, 'rb'))
                        logger.debug(_("Wrote chunk %(chunk_id)d/"
                                       "%(total_chunks)d to Swift "
                                       "returning MD5 of content: "
                                       "%(chunk_etag)s")
                                     % locals())
                    bytes_left -= self.large_object_chunk_size
                    chunk_id += 1

                # Now we write the object manifest and return the
                # manifest's etag...
                manifest = "%s/%s" % (self.container, obj_name)
                headers = {'ETag': hashlib.md5("").hexdigest(),
                           'X-Object-Manifest': manifest}

                # The ETag returned for the manifest is actually the
                # MD5 hash of the concatenated checksums of the strings
                # of each chunk...so we ignore this result in favour of
                # the MD5 of the entire image file contents, so that
                # users can verify the image file contents accordingly
                _ignored = swift_conn.put_object(self.container, obj_name,
                                                 None, headers=headers)
                obj_etag = checksum.hexdigest()

            # NOTE: We return the user and key here! Have to because
            # location is used by the API server to return the actual
            # image data. We *really* should consider NOT returning
            # the location attribute from GET /images/<ID> and
            # GET /images/details

            return (location.get_uri(), image_size, obj_etag)
        except swift_client.ClientException, e:
            if e.http_status == httplib.CONFLICT:
                raise exception.Duplicate(_("Swift already has an image at "
                                          "location %s") % location.get_uri())
            msg = (_("Failed to add object to Swift.\n"
                   "Got error from Swift: %(e)s") % locals())
            logger.error(msg)
            raise glance.store.BackendException(msg)
コード例 #25
0
ファイル: swift.py プロジェクト: young8/openstack-bill
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns an `glance.store.ImageAddResult` object
        containing information about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval `glance.store.ImageAddResult` object
        :raises `glance.common.exception.Duplicate` if the image already
                existed

        Swift writes the image data using the scheme:
            ``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
        where:
            <USER> = ``swift_store_user``
            <KEY> = ``swift_store_key``
            <AUTH_ADDRESS> = ``swift_store_auth_address``
            <CONTAINER> = ``swift_store_container``
            <ID> = The id of the image being added

        :note Swift auth URLs by default use HTTPS. To specify an HTTP
              auth URL, you can specify http://someurl.com for the
              swift_store_auth_address config option

        :note Swift cannot natively/transparently handle objects >5GB
              in size. So, if the image is greater than 5GB, we write
              chunks of image data to Swift and then write an manifest
              to Swift that contains information about the chunks.
        """
        swift_conn = self._make_swift_connection(
            auth_url=self.full_auth_address, user=self.user, key=self.key)

        create_container_if_missing(self.container, swift_conn, self.options)

        obj_name = str(image_id)
        location = StoreLocation({'scheme': self.scheme,
                                  'container': self.container,
                                  'obj': obj_name,
                                  'authurl': self.auth_address,
                                  'user': self.user,
                                  'key': self.key})

        logger.debug(_("Adding image object '%(obj_name)s' "
                       "to Swift") % locals())
        try:
            if image_size < self.large_object_size:
                # image_size == 0 is when we don't know the size
                # of the image. This can occur with older clients
                # that don't inspect the payload size, and we simply
                # try to put the object into Swift and hope for the
                # best...
                obj_etag = swift_conn.put_object(self.container, obj_name,
                                                 image_file)

                if image_size == 0:
                    resp_headers = swift_conn.head_object(self.container,
                                                          obj_name)
                    # header keys are lowercased by Swift
                    if 'content-length' in resp_headers:
                        image_size = int(resp_headers['content-length'])
            else:
                # Write the image into Swift in chunks. We cannot
                # stream chunks of the webob.Request.body_file, unfortunately,
                # so we must write chunks of the body_file into a temporary
                # disk buffer, and then pass this disk buffer to Swift.
                bytes_left = image_size
                chunk_id = 1
                total_chunks = int(math.ceil(
                    float(image_size) / float(self.large_object_chunk_size)))
                checksum = hashlib.md5()
                while bytes_left > 0:
                    with tempfile.NamedTemporaryFile() as disk_buffer:
                        chunk_size = min(self.large_object_chunk_size,
                                         bytes_left)
                        logger.debug(_("Writing %(chunk_size)d bytes for "
                                       "chunk %(chunk_id)d/"
                                       "%(total_chunks)d to disk buffer "
                                       "for image %(image_id)s")
                                     % locals())
                        chunk = image_file.read(chunk_size)
                        checksum.update(chunk)
                        disk_buffer.write(chunk)
                        disk_buffer.flush()
                        logger.debug(_("Writing chunk %(chunk_id)d/"
                                       "%(total_chunks)d to Swift "
                                       "for image %(image_id)s")
                                     % locals())
                        chunk_etag = swift_conn.put_object(
                            self.container,
                            "%s-%05d" % (obj_name, chunk_id),
                            open(disk_buffer.name, 'rb'))
                        logger.debug(_("Wrote chunk %(chunk_id)d/"
                                       "%(total_chunks)d to Swift "
                                       "returning MD5 of content: "
                                       "%(chunk_etag)s")
                                     % locals())
                    bytes_left -= self.large_object_chunk_size
                    chunk_id += 1

                # Now we write the object manifest and return the
                # manifest's etag...
                manifest = "%s/%s" % (self.container, obj_name)
                headers = {'ETag': hashlib.md5("").hexdigest(),
                           'X-Object-Manifest': manifest}

                # The ETag returned for the manifest is actually the
                # MD5 hash of the concatenated checksums of the strings
                # of each chunk...so we ignore this result in favour of
                # the MD5 of the entire image file contents, so that
                # users can verify the image file contents accordingly
                _ignored = swift_conn.put_object(self.container, obj_name,
                                                 None, headers=headers)
                obj_etag = checksum.hexdigest()

            # NOTE: We return the user and key here! Have to because
            # location is used by the API server to return the actual
            # image data. We *really* should consider NOT returning
            # the location attribute from GET /images/<ID> and
            # GET /images/details

            return (location.get_uri(), image_size, obj_etag)
        except swift_client.ClientException, e:
            if e.http_status == httplib.CONFLICT:
                raise exception.Duplicate(_("Swift already has an image at "
                                          "location %s") % location.get_uri())
            msg = (_("Failed to add object to Swift.\n"
                   "Got error from Swift: %(e)s") % locals())
            logger.error(msg)
            raise glance.store.BackendException(msg)
コード例 #26
0
ファイル: swift.py プロジェクト: hanlind/glance
                segments = connection.get_container(
                        obj_container, prefix=obj_prefix)[1]
                for segment in segments:
                    # TODO(jaypipes): This would be an easy area to parallelize
                    # since we're simply sending off parallelizable requests
                    # to Swift to delete stuff. It's not like we're going to
                    # be hogging up network or file I/O here...
                    connection.delete_object(
                            obj_container, segment['name'])

            else:
                connection.delete_object(location.container, location.obj)

        except swiftclient.ClientException, e:
            if e.http_status == httplib.NOT_FOUND:
                uri = location.get_uri()
                raise exception.NotFound(_("Swift could not find image at "
                                           "uri %(uri)s") % locals())
            else:
                raise

    def _create_container_if_missing(self, container, connection):
        """
        Creates a missing container in Swift if the
        ``swift_store_create_container_on_put`` option is set.

        :param container: Name of container to create
        :param connection: Connection to swift service
        """
        try:
            connection.head_container(container)