示例#1
0
    def create(self, stream):
        sysmeta = {}
        sysmeta['id'] = self.content_id
        sysmeta['version'] = self.version
        sysmeta['policy'] = self.stgpol
        sysmeta['mime_type'] = self.mime_type
        sysmeta['chunk_method'] = self.chunk_method
        sysmeta['chunk_size'] = self.metadata['chunk_size']

        chunks = _sort_chunks(self.chunks.raw(), self.storage_method.ec)
        sysmeta['content_path'] = self.path
        sysmeta['container_id'] = self.container_id

        headers = {}
        handler = ECWriteHandler(stream,
                                 sysmeta,
                                 chunks,
                                 self.storage_method,
                                 headers=headers)

        final_chunks, bytes_transferred, content_checksum = handler.stream()

        # TODO sanity checks

        self.checksum = content_checksum
        self._create_object()
        return final_chunks, bytes_transferred, content_checksum
示例#2
0
    def _object_create(self, account, container, obj_name, source,
                       sysmeta, metadata=None, policy=None, headers=None,
                       application_key=None):
        meta, raw_chunks = self._content_prepare(
            account, container, obj_name, sysmeta['content_length'],
            policy=policy, headers=headers)

        sysmeta['chunk_size'] = int(meta['X-oio-ns-chunk-size'])
        sysmeta['id'] = meta[object_headers['id']]
        sysmeta['version'] = meta[object_headers['version']]
        sysmeta['policy'] = meta[object_headers['policy']]
        sysmeta['mime_type'] = meta[object_headers['mime_type']]
        sysmeta['chunk_method'] = meta[object_headers['chunk_method']]

        storage_method = STORAGE_METHODS.load(sysmeta['chunk_method'])

        chunks = _sort_chunks(raw_chunks, storage_method.ec)
        sysmeta['content_path'] = obj_name
        sysmeta['container_id'] = utils.name2cid(account, container)
        sysmeta['ns'] = self.namespace

        if storage_method.ec:
            handler = ECWriteHandler(source, sysmeta, chunks, storage_method,
                                     headers=headers)
        elif storage_method.backblaze:
            backblaze_info = self._put_meta_backblaze(storage_method,
                                                      application_key)
            handler = BackblazeWriteHandler(source, sysmeta,
                                            chunks, storage_method,
                                            headers, backblaze_info)
        else:
            handler = ReplicatedWriteHandler(source, sysmeta, chunks,
                                             storage_method, headers=headers)

        final_chunks, bytes_transferred, content_checksum = handler.stream()

        etag = sysmeta['etag']
        if etag and etag.lower() != content_checksum.lower():
            raise exc.EtagMismatch(
                "given etag %s != computed %s" % (etag, content_checksum))
        sysmeta['etag'] = content_checksum

        h = {}
        h[object_headers['size']] = bytes_transferred
        h[object_headers['hash']] = sysmeta['etag']
        h[object_headers['version']] = sysmeta['version']
        h[object_headers['id']] = sysmeta['id']
        h[object_headers['policy']] = sysmeta['policy']
        h[object_headers['mime_type']] = sysmeta['mime_type']
        h[object_headers['chunk_method']] = sysmeta['chunk_method']

        if metadata:
            for k, v in metadata.iteritems():
                h['%sx-%s' % (constants.OBJECT_METADATA_PREFIX, k)] = v

        m, body = self._content_create(account, container, obj_name,
                                       final_chunks, headers=h)
        return final_chunks, bytes_transferred, content_checksum
示例#3
0
    def _object_create(self, account, container, obj_name, source,
                       sysmeta, properties=None, policy=None,
                       key_file=None, **kwargs):
        self._patch_timeouts(kwargs)
        obj_meta, chunk_prep = self._content_preparer(
            account, container, obj_name,
            policy=policy, **kwargs)
        obj_meta.update(sysmeta)
        obj_meta['content_path'] = obj_name
        obj_meta['container_id'] = cid_from_name(account, container).upper()
        obj_meta['ns'] = self.namespace
        obj_meta['full_path'] = self._generate_fullpath(account, container,
                                                        obj_name,
                                                        obj_meta['version'])
        obj_meta['oio_version'] = (obj_meta.get('oio_version')
                                   or OIO_VERSION)

        # XXX content_id is necessary to update an existing object
        kwargs['content_id'] = kwargs.get('content_id', obj_meta['id'])

        storage_method = STORAGE_METHODS.load(obj_meta['chunk_method'])
        if storage_method.ec:
            handler = ECWriteHandler(
                source, obj_meta, chunk_prep, storage_method, **kwargs)
        elif storage_method.backblaze:
            backblaze_info = self._b2_credentials(storage_method, key_file)
            handler = BackblazeWriteHandler(
                source, obj_meta, chunk_prep, storage_method,
                backblaze_info, **kwargs)
        else:
            handler = ReplicatedWriteHandler(
                source, obj_meta, chunk_prep, storage_method, **kwargs)

        final_chunks, bytes_transferred, content_checksum = handler.stream()

        etag = obj_meta.get('etag')
        if etag and etag.lower() != content_checksum.lower():
            raise exc.EtagMismatch(
                "given etag %s != computed %s" % (etag, content_checksum))
        obj_meta['etag'] = content_checksum

        data = {'chunks': final_chunks, 'properties': properties or {}}
        # FIXME: we may just pass **obj_meta
        self.container.content_create(
            account, container, obj_name, size=bytes_transferred,
            checksum=content_checksum, data=data,
            stgpol=obj_meta['policy'],
            version=obj_meta['version'], mime_type=obj_meta['mime_type'],
            chunk_method=obj_meta['chunk_method'],
            **kwargs)
        return final_chunks, bytes_transferred, content_checksum
示例#4
0
    def create(self, stream, **kwargs):
        sysmeta = self._generate_sysmeta()
        chunks = _sort_chunks(self.chunks.raw(), self.storage_method.ec)

        headers = {}
        handler = ECWriteHandler(
            stream, sysmeta, chunks, self.storage_method, headers=headers)

        final_chunks, bytes_transferred, content_checksum = handler.stream()

        # TODO sanity checks

        self.checksum = content_checksum
        self._create_object(**kwargs)
        return final_chunks, bytes_transferred, content_checksum
示例#5
0
    def _object_create(
        self, account, container, obj_name, source, sysmeta, metadata=None, policy=None, headers=None, key_file=None
    ):
        meta, chunk_prep = self._content_preparer(account, container, obj_name, policy=policy, headers=headers)
        sysmeta["chunk_size"] = int(meta["X-oio-ns-chunk-size"])
        sysmeta["id"] = meta[object_headers["id"]]
        sysmeta["version"] = meta[object_headers["version"]]
        sysmeta["policy"] = meta[object_headers["policy"]]
        if not sysmeta.get("mime_type"):
            sysmeta["mime_type"] = meta[object_headers["mime_type"]]
        sysmeta["chunk_method"] = meta[object_headers["chunk_method"]]
        sysmeta["content_path"] = obj_name
        sysmeta["container_id"] = utils.name2cid(account, container).upper()
        sysmeta["ns"] = self.namespace

        storage_method = STORAGE_METHODS.load(sysmeta["chunk_method"])
        if storage_method.ec:
            handler = ECWriteHandler(source, sysmeta, chunk_prep, storage_method, headers=headers)
        elif storage_method.backblaze:
            backblaze_info = self._b2_credentials(storage_method, key_file)
            handler = BackblazeWriteHandler(source, sysmeta, chunk_prep, storage_method, headers, backblaze_info)
        else:
            handler = ReplicatedWriteHandler(source, sysmeta, chunk_prep, storage_method, headers=headers)

        final_chunks, bytes_transferred, content_checksum = handler.stream()

        etag = sysmeta["etag"]
        if etag and etag.lower() != content_checksum.lower():
            raise exc.EtagMismatch("given etag %s != computed %s" % (etag, content_checksum))
        sysmeta["etag"] = content_checksum

        h = dict()
        h.update(headers)
        h[object_headers["size"]] = bytes_transferred
        h[object_headers["hash"]] = sysmeta["etag"]
        h[object_headers["version"]] = sysmeta["version"]
        h[object_headers["id"]] = sysmeta["id"]
        h[object_headers["policy"]] = sysmeta["policy"]
        h[object_headers["mime_type"]] = sysmeta["mime_type"]
        h[object_headers["chunk_method"]] = sysmeta["chunk_method"]

        m, body = self._content_create(account, container, obj_name, final_chunks, metadata=metadata, headers=h)
        return final_chunks, bytes_transferred, content_checksum
示例#6
0
文件: ec.py 项目: fvennetier/oio-sds
    def create(self, stream):
        sysmeta = {}
        sysmeta["id"] = self.content_id
        sysmeta["version"] = self.version
        sysmeta["policy"] = self.stgpol
        sysmeta["mime_type"] = self.mime_type
        sysmeta["chunk_method"] = self.chunk_method
        sysmeta["chunk_size"] = self.metadata["chunk-size"]

        chunks = _sort_chunks(self.chunks.raw(), self.storage_method.ec)
        sysmeta["content_path"] = self.path
        sysmeta["container_id"] = self.container_id

        headers = {}
        handler = ECWriteHandler(stream, sysmeta, chunks, self.storage_method, headers=headers)

        final_chunks, bytes_transferred, content_checksum = handler.stream()

        # TODO sanity checks

        self.checksum = content_checksum
        self._create_object()
        return final_chunks, bytes_transferred, content_checksum
示例#7
0
    def _object_create(self,
                       account,
                       container,
                       obj_name,
                       source,
                       sysmeta,
                       properties=None,
                       policy=None,
                       headers=None,
                       key_file=None):
        obj_meta, chunk_prep = self._content_preparer(account,
                                                      container,
                                                      obj_name,
                                                      policy=policy,
                                                      headers=headers)
        obj_meta.update(sysmeta)
        obj_meta['content_path'] = obj_name
        obj_meta['container_id'] = utils.name2cid(account, container).upper()
        obj_meta['ns'] = self.namespace

        storage_method = STORAGE_METHODS.load(obj_meta['chunk_method'])
        if storage_method.ec:
            handler = ECWriteHandler(
                source,
                obj_meta,
                chunk_prep,
                storage_method,
                headers=headers,
                write_timeout=self.write_timeout,
                read_timeout=self.read_timeout,
                connection_timeout=self.connection_timeout)
        elif storage_method.backblaze:
            backblaze_info = self._b2_credentials(storage_method, key_file)
            handler = BackblazeWriteHandler(source, obj_meta, chunk_prep,
                                            storage_method, headers,
                                            backblaze_info)
        else:
            handler = ReplicatedWriteHandler(
                source,
                obj_meta,
                chunk_prep,
                storage_method,
                headers=headers,
                write_timeout=self.write_timeout,
                read_timeout=self.read_timeout,
                connection_timeout=self.connection_timeout)

        final_chunks, bytes_transferred, content_checksum = handler.stream()

        etag = obj_meta.get('etag')
        if etag and etag.lower() != content_checksum.lower():
            raise exc.EtagMismatch("given etag %s != computed %s" %
                                   (etag, content_checksum))
        obj_meta['etag'] = content_checksum

        data = {'chunks': final_chunks, 'properties': properties or {}}
        # FIXME: we may just pass **obj_meta
        self.container.content_create(account,
                                      container,
                                      obj_name,
                                      size=bytes_transferred,
                                      checksum=content_checksum,
                                      data=data,
                                      content_id=obj_meta['id'],
                                      stgpol=obj_meta['policy'],
                                      version=obj_meta['version'],
                                      mime_type=obj_meta['mime_type'],
                                      chunk_method=obj_meta['chunk_method'],
                                      headers=headers)
        return final_chunks, bytes_transferred, content_checksum