コード例 #1
0
ファイル: rbd.py プロジェクト: wendy-king/x7_client_venv
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns an `tank.store.ImageAddResult` object
        containing information about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval `tank.store.ImageAddResult` object
        :raises `tank.common.exception.Duplicate` if the image already
                existed
        """
        location = StoreLocation({'image': image_id})
        checksum = hashlib.md5()
        image_name = str(image_id)
        with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
            with conn.open_ioctx(self.pool) as ioctx:
                order = int(math.log(self.chunk_size, 2))
                logger.debug('creating image %s with order %d',
                             image_name, order)
                try:
                    rbd.RBD().create(ioctx, image_name, image_size, order)
                except rbd.ImageExists:
                    raise exception.Duplicate(
                        _('RBD image %s already exists') % image_id)
                with rbd.Image(ioctx, image_name) as image:
                    bytes_left = image_size
                    while bytes_left > 0:
                        length = min(self.chunk_size, bytes_left)
                        data = image_file.read(length)
                        image.write(data, image_size - bytes_left)
                        bytes_left -= length
                        checksum.update(data)

        return (location.get_uri(), image_size, checksum.hexdigest())
コード例 #2
0
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns an `tank.store.ImageAddResult` object
        containing information about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval `tank.store.ImageAddResult` object
        :raises `tank.common.exception.Duplicate` if the image already
                existed
        """
        location = StoreLocation({'image': image_id})
        checksum = hashlib.md5()
        image_name = str(image_id)
        with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
            with conn.open_ioctx(self.pool) as ioctx:
                order = int(math.log(self.chunk_size, 2))
                logger.debug('creating image %s with order %d', image_name,
                             order)
                try:
                    rbd.RBD().create(ioctx, image_name, image_size, order)
                except rbd.ImageExists:
                    raise exception.Duplicate(
                        _('RBD image %s already exists') % image_id)
                with rbd.Image(ioctx, image_name) as image:
                    bytes_left = image_size
                    while bytes_left > 0:
                        length = min(self.chunk_size, bytes_left)
                        data = image_file.read(length)
                        image.write(data, image_size - bytes_left)
                        bytes_left -= length
                        checksum.update(data)

        return (location.get_uri(), image_size, checksum.hexdigest())
コード例 #3
0
ファイル: chase.py プロジェクト: wendy-king/x7_client_venv
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns an `tank.store.ImageAddResult` object
        containing information about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval `tank.store.ImageAddResult` object
        :raises `tank.common.exception.Duplicate` if the image already
                existed

        Chase writes the image data using the scheme:
            ``chase://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
        where:
            <USER> = ``chase_store_user``
            <KEY> = ``chase_store_key``
            <AUTH_ADDRESS> = ``chase_store_auth_address``
            <CONTAINER> = ``chase_store_container``
            <ID> = The id of the image being added

        :note Chase auth URLs by default use HTTPS. To specify an HTTP
              auth URL, you can specify http://someurl.com for the
              chase_store_auth_address config option

        :note Chase cannot natively/transparently handle objects >5GB
              in size. So, if the image is greater than 5GB, we write
              chunks of image data to Chase and then write an manifest
              to Chase that contains information about the chunks.
              This same chunking process is used by default for images
              of an unknown size, as pushing them directly to chase would
              fail if the image turns out to be greater than 5GB.
        """
        chase_conn = self._make_chase_connection(
            auth_url=self.full_auth_address, user=self.user, key=self.key)

        create_container_if_missing(self.container, chase_conn, self.conf)

        obj_name = str(image_id)
        location = StoreLocation({'scheme': self.scheme,
                                  'container': self.container,
                                  'obj': obj_name,
                                  'authurl': self.auth_address,
                                  'user': self.user,
                                  'key': self.key})

        logger.debug(_("Adding image object '%(obj_name)s' "
                       "to Chase") % locals())
        try:
            if image_size > 0 and image_size < self.large_object_size:
                # Image size is known, and is less than large_object_size.
                # Send to Chase with regular PUT.
                obj_etag = chase_conn.put_object(self.container, obj_name,
                                                 image_file,
                                                 content_length=image_size)
            else:
                # Write the image into Chase in chunks.
                chunk_id = 1
                if image_size > 0:
                    total_chunks = str(int(
                        math.ceil(float(image_size) /
                                  float(self.large_object_chunk_size))))
                else:
                    # image_size == 0 is when we don't know the size
                    # of the image. This can occur with older clients
                    # that don't inspect the payload size.
                    logger.debug(_("Cannot determine image size. Adding as a "
                                   "segmented object to Chase."))
                    total_chunks = '?'

                checksum = hashlib.md5()
                combined_chunks_size = 0
                while True:
                    chunk_size = self.large_object_chunk_size
                    if image_size == 0:
                        content_length = None
                    else:
                        left = image_size - combined_chunks_size
                        if left == 0:
                            break
                        if chunk_size > left:
                            chunk_size = left
                        content_length = chunk_size

                    chunk_name = "%s-%05d" % (obj_name, chunk_id)
                    reader = ChunkReader(image_file, checksum, chunk_size)
                    chunk_etag = chase_conn.put_object(
                        self.container, chunk_name, reader,
                        content_length=content_length)
                    bytes_read = reader.bytes_read
                    logger.debug(_("Wrote chunk %(chunk_id)d/"
                                   "%(total_chunks)s of length %(bytes_read)d "
                                   "to Chase returning MD5 of content: "
                                   "%(chunk_etag)s")
                                 % locals())

                    if bytes_read == 0:
                        # Delete the last chunk, because it's of zero size.
                        # This will happen if image_size == 0.
                        logger.debug(_("Deleting final zero-length chunk"))
                        chase_conn.delete_object(self.container, chunk_name)
                        break

                    chunk_id += 1
                    combined_chunks_size += bytes_read

                # In the case we have been given an unknown image size,
                # set the image_size to the total size of the combined chunks.
                if image_size == 0:
                    image_size = combined_chunks_size

                # Now we write the object manifest and return the
                # manifest's etag...
                manifest = "%s/%s" % (self.container, obj_name)
                headers = {'ETag': hashlib.md5("").hexdigest(),
                           'X-Object-Manifest': manifest}

                # The ETag returned for the manifest is actually the
                # MD5 hash of the concatenated checksums of the strings
                # of each chunk...so we ignore this result in favour of
                # the MD5 of the entire image file contents, so that
                # users can verify the image file contents accordingly
                _ignored = chase_conn.put_object(self.container, obj_name,
                                                 None, headers=headers)
                obj_etag = checksum.hexdigest()

            # NOTE: We return the user and key here! Have to because
            # location is used by the API server to return the actual
            # image data. We *really* should consider NOT returning
            # the location attribute from GET /images/<ID> and
            # GET /images/details

            return (location.get_uri(), image_size, obj_etag)
        except chase_client.ClientException, e:
            if e.http_status == httplib.CONFLICT:
                raise exception.Duplicate(_("Chase already has an image at "
                                          "location %s") % location.get_uri())
            msg = (_("Failed to add object to Chase.\n"
                   "Got error from Chase: %(e)s") % locals())
            logger.error(msg)
            raise tank.store.BackendException(msg)
コード例 #4
0
ファイル: chase.py プロジェクト: wendy-king/x7_venv
    def add(self, image_id, image_file, image_size):
        """
        Stores an image file with supplied identifier to the backend
        storage system and returns an `tank.store.ImageAddResult` object
        containing information about the stored image.

        :param image_id: The opaque image identifier
        :param image_file: The image data to write, as a file-like object
        :param image_size: The size of the image data to write, in bytes

        :retval `tank.store.ImageAddResult` object
        :raises `tank.common.exception.Duplicate` if the image already
                existed

        Chase writes the image data using the scheme:
            ``chase://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
        where:
            <USER> = ``chase_store_user``
            <KEY> = ``chase_store_key``
            <AUTH_ADDRESS> = ``chase_store_auth_address``
            <CONTAINER> = ``chase_store_container``
            <ID> = The id of the image being added

        :note Chase auth URLs by default use HTTPS. To specify an HTTP
              auth URL, you can specify http://someurl.com for the
              chase_store_auth_address config option

        :note Chase cannot natively/transparently handle objects >5GB
              in size. So, if the image is greater than 5GB, we write
              chunks of image data to Chase and then write an manifest
              to Chase that contains information about the chunks.
              This same chunking process is used by default for images
              of an unknown size, as pushing them directly to chase would
              fail if the image turns out to be greater than 5GB.
        """
        chase_conn = self._make_chase_connection(
            auth_url=self.full_auth_address, user=self.user, key=self.key)

        create_container_if_missing(self.container, chase_conn, self.conf)

        obj_name = str(image_id)
        location = StoreLocation({
            'scheme': self.scheme,
            'container': self.container,
            'obj': obj_name,
            'authurl': self.auth_address,
            'user': self.user,
            'key': self.key
        })

        logger.debug(
            _("Adding image object '%(obj_name)s' "
              "to Chase") % locals())
        try:
            if image_size > 0 and image_size < self.large_object_size:
                # Image size is known, and is less than large_object_size.
                # Send to Chase with regular PUT.
                obj_etag = chase_conn.put_object(self.container,
                                                 obj_name,
                                                 image_file,
                                                 content_length=image_size)
            else:
                # Write the image into Chase in chunks.
                chunk_id = 1
                if image_size > 0:
                    total_chunks = str(
                        int(
                            math.ceil(
                                float(image_size) /
                                float(self.large_object_chunk_size))))
                else:
                    # image_size == 0 is when we don't know the size
                    # of the image. This can occur with older clients
                    # that don't inspect the payload size.
                    logger.debug(
                        _("Cannot determine image size. Adding as a "
                          "segmented object to Chase."))
                    total_chunks = '?'

                checksum = hashlib.md5()
                combined_chunks_size = 0
                while True:
                    chunk_size = self.large_object_chunk_size
                    if image_size == 0:
                        content_length = None
                    else:
                        left = image_size - combined_chunks_size
                        if left == 0:
                            break
                        if chunk_size > left:
                            chunk_size = left
                        content_length = chunk_size

                    chunk_name = "%s-%05d" % (obj_name, chunk_id)
                    reader = ChunkReader(image_file, checksum, chunk_size)
                    chunk_etag = chase_conn.put_object(
                        self.container,
                        chunk_name,
                        reader,
                        content_length=content_length)
                    bytes_read = reader.bytes_read
                    logger.debug(
                        _("Wrote chunk %(chunk_id)d/"
                          "%(total_chunks)s of length %(bytes_read)d "
                          "to Chase returning MD5 of content: "
                          "%(chunk_etag)s") % locals())

                    if bytes_read == 0:
                        # Delete the last chunk, because it's of zero size.
                        # This will happen if image_size == 0.
                        logger.debug(_("Deleting final zero-length chunk"))
                        chase_conn.delete_object(self.container, chunk_name)
                        break

                    chunk_id += 1
                    combined_chunks_size += bytes_read

                # In the case we have been given an unknown image size,
                # set the image_size to the total size of the combined chunks.
                if image_size == 0:
                    image_size = combined_chunks_size

                # Now we write the object manifest and return the
                # manifest's etag...
                manifest = "%s/%s" % (self.container, obj_name)
                headers = {
                    'ETag': hashlib.md5("").hexdigest(),
                    'X-Object-Manifest': manifest
                }

                # The ETag returned for the manifest is actually the
                # MD5 hash of the concatenated checksums of the strings
                # of each chunk...so we ignore this result in favour of
                # the MD5 of the entire image file contents, so that
                # users can verify the image file contents accordingly
                _ignored = chase_conn.put_object(self.container,
                                                 obj_name,
                                                 None,
                                                 headers=headers)
                obj_etag = checksum.hexdigest()

            # NOTE: We return the user and key here! Have to because
            # location is used by the API server to return the actual
            # image data. We *really* should consider NOT returning
            # the location attribute from GET /images/<ID> and
            # GET /images/details

            return (location.get_uri(), image_size, obj_etag)
        except chase_client.ClientException, e:
            if e.http_status == httplib.CONFLICT:
                raise exception.Duplicate(
                    _("Chase already has an image at "
                      "location %s") % location.get_uri())
            msg = (_("Failed to add object to Chase.\n"
                     "Got error from Chase: %(e)s") % locals())
            logger.error(msg)
            raise tank.store.BackendException(msg)