Esempio n. 1
0
 def object_fetch(self,
                  account,
                  container,
                  obj,
                  version=None,
                  ranges=None,
                  key_file=None,
                  **kwargs):
     meta, raw_chunks = self.object_locate(account,
                                           container,
                                           obj,
                                           version=version,
                                           **kwargs)
     chunk_method = meta['chunk_method']
     storage_method = STORAGE_METHODS.load(chunk_method)
     chunks = _sort_chunks(raw_chunks, storage_method.ec)
     meta['container_id'] = name2cid(account, container).upper()
     meta['ns'] = self.namespace
     self._patch_timeouts(kwargs)
     if storage_method.ec:
         stream = fetch_stream_ec(chunks, ranges, storage_method, **kwargs)
     elif storage_method.backblaze:
         stream = self._fetch_stream_backblaze(meta, chunks, ranges,
                                               storage_method, key_file,
                                               **kwargs)
     else:
         stream = fetch_stream(chunks, ranges, storage_method, **kwargs)
     return meta, stream
Esempio n. 2
0
 def object_fetch(self,
                  account,
                  container,
                  obj,
                  ranges=None,
                  headers=None,
                  key_file=None):
     if not headers:
         headers = dict()
     if 'X-oio-req-id' not in headers:
         headers['X-oio-req-id'] = utils.request_id()
     meta, raw_chunks = self.object_locate(account,
                                           container,
                                           obj,
                                           headers=headers)
     chunk_method = meta['chunk_method']
     storage_method = STORAGE_METHODS.load(chunk_method)
     chunks = _sort_chunks(raw_chunks, storage_method.ec)
     meta['container_id'] = utils.name2cid(account, container).upper()
     meta['ns'] = self.namespace
     if storage_method.ec:
         stream = self._fetch_stream_ec(meta, chunks, ranges,
                                        storage_method, headers)
     elif storage_method.backblaze:
         stream = self._fetch_stream_backblaze(meta, chunks, ranges,
                                               storage_method, key_file)
     else:
         stream = self._fetch_stream(meta, chunks, ranges, storage_method,
                                     headers)
     return meta, stream
Esempio n. 3
0
    def _object_create(self, account, container, obj_name, source,
                       sysmeta, metadata=None, policy=None, headers=None,
                       application_key=None):
        meta, raw_chunks = self._content_prepare(
            account, container, obj_name, sysmeta['content_length'],
            policy=policy, headers=headers)

        sysmeta['chunk_size'] = int(meta['X-oio-ns-chunk-size'])
        sysmeta['id'] = meta[object_headers['id']]
        sysmeta['version'] = meta[object_headers['version']]
        sysmeta['policy'] = meta[object_headers['policy']]
        sysmeta['mime_type'] = meta[object_headers['mime_type']]
        sysmeta['chunk_method'] = meta[object_headers['chunk_method']]

        storage_method = STORAGE_METHODS.load(sysmeta['chunk_method'])

        chunks = _sort_chunks(raw_chunks, storage_method.ec)
        sysmeta['content_path'] = obj_name
        sysmeta['container_id'] = utils.name2cid(account, container)
        sysmeta['ns'] = self.namespace

        if storage_method.ec:
            handler = ECWriteHandler(source, sysmeta, chunks, storage_method,
                                     headers=headers)
        elif storage_method.backblaze:
            backblaze_info = self._put_meta_backblaze(storage_method,
                                                      application_key)
            handler = BackblazeWriteHandler(source, sysmeta,
                                            chunks, storage_method,
                                            headers, backblaze_info)
        else:
            handler = ReplicatedWriteHandler(source, sysmeta, chunks,
                                             storage_method, headers=headers)

        final_chunks, bytes_transferred, content_checksum = handler.stream()

        etag = sysmeta['etag']
        if etag and etag.lower() != content_checksum.lower():
            raise exc.EtagMismatch(
                "given etag %s != computed %s" % (etag, content_checksum))
        sysmeta['etag'] = content_checksum

        h = {}
        h[object_headers['size']] = bytes_transferred
        h[object_headers['hash']] = sysmeta['etag']
        h[object_headers['version']] = sysmeta['version']
        h[object_headers['id']] = sysmeta['id']
        h[object_headers['policy']] = sysmeta['policy']
        h[object_headers['mime_type']] = sysmeta['mime_type']
        h[object_headers['chunk_method']] = sysmeta['chunk_method']

        if metadata:
            for k, v in metadata.iteritems():
                h['%sx-%s' % (constants.OBJECT_METADATA_PREFIX, k)] = v

        m, body = self._content_create(account, container, obj_name,
                                       final_chunks, headers=h)
        return final_chunks, bytes_transferred, content_checksum
 def object_fetch(self, account, container, obj, ranges=None, headers=None, key_file=None):
     if not headers:
         headers = dict()
     if "X-oio-req-id" not in headers:
         headers["X-oio-req-id"] = utils.request_id()
     meta, raw_chunks = self.object_analyze(account, container, obj, headers=headers)
     chunk_method = meta["chunk-method"]
     storage_method = STORAGE_METHODS.load(chunk_method)
     chunks = _sort_chunks(raw_chunks, storage_method.ec)
     meta["container_id"] = utils.name2cid(account, container).upper()
     meta["ns"] = self.namespace
     if storage_method.ec:
         stream = self._fetch_stream_ec(meta, chunks, ranges, storage_method, headers)
     elif storage_method.backblaze:
         stream = self._fetch_stream_backblaze(meta, chunks, ranges, storage_method, key_file)
     else:
         stream = self._fetch_stream(meta, chunks, ranges, storage_method, headers)
     return meta, stream
Esempio n. 5
0
 def object_delete(self, account, container, obj, headers={},
                   application_key=None):
     uri = self._make_uri('content/delete')
     params = self._make_params(account, container, obj)
     meta, raw_chunks = self.object_analyze(
         account, container, obj, headers=headers)
     if meta:
         chunk_method = meta['chunk-method']
         storage_method = STORAGE_METHODS.load(chunk_method)
         meta['ns'] = self.namespace
         meta['container_id'] = utils.name2cid(account, container)
         chunks = _sort_chunks(raw_chunks, storage_method.ec)
         if storage_method.backblaze:
             backblaze_info = self._put_meta_backblaze(storage_method,
                                                       application_key)
             BackblazeDeleteHandler(meta, chunks, backblaze_info).delete()
     resp, resp_body = self._request(
         'POST', uri, params=params, headers=headers)
Esempio n. 6
0
 def object_fetch(self, account, container, obj, ranges=None,
                  headers=None, key_file=None):
     meta, raw_chunks = self.object_analyze(
         account, container, obj, headers=headers)
     chunk_method = meta['chunk-method']
     storage_method = STORAGE_METHODS.load(chunk_method)
     chunks = _sort_chunks(raw_chunks, storage_method.ec)
     meta['container_id'] = utils.name2cid(account, container).upper()
     meta['ns'] = self.namespace
     if storage_method.ec:
         stream = self._fetch_stream_ec(meta, chunks, ranges,
                                        storage_method, headers)
     elif storage_method.backblaze:
         stream = self._fetch_stream_backblaze(meta, chunks, ranges,
                                               storage_method, key_file)
     else:
         stream = self._fetch_stream(meta, chunks, ranges, storage_method,
                                     headers)
     return meta, stream
Esempio n. 7
0
    def _object_create(
        self, account, container, obj_name, source, sysmeta, metadata=None, policy=None, headers=None, key_file=None
    ):
        meta, chunk_prep = self._content_preparer(account, container, obj_name, policy=policy, headers=headers)
        sysmeta["chunk_size"] = int(meta["X-oio-ns-chunk-size"])
        sysmeta["id"] = meta[object_headers["id"]]
        sysmeta["version"] = meta[object_headers["version"]]
        sysmeta["policy"] = meta[object_headers["policy"]]
        if not sysmeta.get("mime_type"):
            sysmeta["mime_type"] = meta[object_headers["mime_type"]]
        sysmeta["chunk_method"] = meta[object_headers["chunk_method"]]
        sysmeta["content_path"] = obj_name
        sysmeta["container_id"] = utils.name2cid(account, container).upper()
        sysmeta["ns"] = self.namespace

        storage_method = STORAGE_METHODS.load(sysmeta["chunk_method"])
        if storage_method.ec:
            handler = ECWriteHandler(source, sysmeta, chunk_prep, storage_method, headers=headers)
        elif storage_method.backblaze:
            backblaze_info = self._b2_credentials(storage_method, key_file)
            handler = BackblazeWriteHandler(source, sysmeta, chunk_prep, storage_method, headers, backblaze_info)
        else:
            handler = ReplicatedWriteHandler(source, sysmeta, chunk_prep, storage_method, headers=headers)

        final_chunks, bytes_transferred, content_checksum = handler.stream()

        etag = sysmeta["etag"]
        if etag and etag.lower() != content_checksum.lower():
            raise exc.EtagMismatch("given etag %s != computed %s" % (etag, content_checksum))
        sysmeta["etag"] = content_checksum

        h = dict()
        h.update(headers)
        h[object_headers["size"]] = bytes_transferred
        h[object_headers["hash"]] = sysmeta["etag"]
        h[object_headers["version"]] = sysmeta["version"]
        h[object_headers["id"]] = sysmeta["id"]
        h[object_headers["policy"]] = sysmeta["policy"]
        h[object_headers["mime_type"]] = sysmeta["mime_type"]
        h[object_headers["chunk_method"]] = sysmeta["chunk_method"]

        m, body = self._content_create(account, container, obj_name, final_chunks, metadata=metadata, headers=h)
        return final_chunks, bytes_transferred, content_checksum
Esempio n. 8
0
 def object_fetch(self, account, container, obj, ranges=None,
                  headers=None, application_key=None):
     meta, raw_chunks = self.object_analyze(
         account, container, obj, headers=headers)
     chunk_method = meta['chunk-method']
     storage_method = STORAGE_METHODS.load(chunk_method)
     chunks = _sort_chunks(raw_chunks, storage_method.ec)
     meta['container_id'] = utils.name2cid(account, container)
     meta['ns'] = self.namespace
     if storage_method.ec:
         stream = self._fetch_stream_ec(meta, chunks, ranges,
                                        storage_method, headers)
     elif storage_method.backblaze:
         stream = self._fetch_stream_backblaze(meta, chunks, ranges,
                                               storage_method,
                                               application_key)
     else:
         stream = self._fetch_stream(meta, chunks, ranges, storage_method,
                                     headers)
     return meta, stream
Esempio n. 9
0
    def _object_create(self,
                       account,
                       container,
                       obj_name,
                       source,
                       sysmeta,
                       properties=None,
                       policy=None,
                       headers=None,
                       key_file=None):
        obj_meta, chunk_prep = self._content_preparer(account,
                                                      container,
                                                      obj_name,
                                                      policy=policy,
                                                      headers=headers)
        obj_meta.update(sysmeta)
        obj_meta['content_path'] = obj_name
        obj_meta['container_id'] = utils.name2cid(account, container).upper()
        obj_meta['ns'] = self.namespace

        storage_method = STORAGE_METHODS.load(obj_meta['chunk_method'])
        if storage_method.ec:
            handler = ECWriteHandler(
                source,
                obj_meta,
                chunk_prep,
                storage_method,
                headers=headers,
                write_timeout=self.write_timeout,
                read_timeout=self.read_timeout,
                connection_timeout=self.connection_timeout)
        elif storage_method.backblaze:
            backblaze_info = self._b2_credentials(storage_method, key_file)
            handler = BackblazeWriteHandler(source, obj_meta, chunk_prep,
                                            storage_method, headers,
                                            backblaze_info)
        else:
            handler = ReplicatedWriteHandler(
                source,
                obj_meta,
                chunk_prep,
                storage_method,
                headers=headers,
                write_timeout=self.write_timeout,
                read_timeout=self.read_timeout,
                connection_timeout=self.connection_timeout)

        final_chunks, bytes_transferred, content_checksum = handler.stream()

        etag = obj_meta.get('etag')
        if etag and etag.lower() != content_checksum.lower():
            raise exc.EtagMismatch("given etag %s != computed %s" %
                                   (etag, content_checksum))
        obj_meta['etag'] = content_checksum

        data = {'chunks': final_chunks, 'properties': properties or {}}
        # FIXME: we may just pass **obj_meta
        self.container.content_create(account,
                                      container,
                                      obj_name,
                                      size=bytes_transferred,
                                      checksum=content_checksum,
                                      data=data,
                                      content_id=obj_meta['id'],
                                      stgpol=obj_meta['policy'],
                                      version=obj_meta['version'],
                                      mime_type=obj_meta['mime_type'],
                                      chunk_method=obj_meta['chunk_method'],
                                      headers=headers)
        return final_chunks, bytes_transferred, content_checksum
Esempio n. 10
0
    def _object_create(self,
                       account,
                       container,
                       obj_name,
                       source,
                       sysmeta,
                       properties=None,
                       policy=None,
                       key_file=None,
                       **kwargs):
        self._patch_timeouts(kwargs)
        obj_meta, chunk_prep = self._content_preparer(account,
                                                      container,
                                                      obj_name,
                                                      policy=policy,
                                                      **kwargs)
        obj_meta.update(sysmeta)
        obj_meta['content_path'] = obj_name
        obj_meta['container_id'] = name2cid(account, container).upper()
        obj_meta['ns'] = self.namespace
        obj_meta['full_path'] = self._generate_fullpath(
            account, container, obj_name, obj_meta['version'])
        obj_meta['oio_version'] = (obj_meta.get('oio_version') or OIO_VERSION)

        # XXX content_id is necessary to update an existing object
        kwargs['content_id'] = kwargs.get('content_id', obj_meta['id'])

        storage_method = STORAGE_METHODS.load(obj_meta['chunk_method'])
        if storage_method.ec:
            handler = ECWriteHandler(source, obj_meta, chunk_prep,
                                     storage_method, **kwargs)
        elif storage_method.backblaze:
            backblaze_info = self._b2_credentials(storage_method, key_file)
            handler = BackblazeWriteHandler(source, obj_meta, chunk_prep,
                                            storage_method, backblaze_info,
                                            **kwargs)
        else:
            handler = ReplicatedWriteHandler(source, obj_meta, chunk_prep,
                                             storage_method, **kwargs)

        final_chunks, bytes_transferred, content_checksum = handler.stream()

        etag = obj_meta.get('etag')
        if etag and etag.lower() != content_checksum.lower():
            raise exc.EtagMismatch("given etag %s != computed %s" %
                                   (etag, content_checksum))
        obj_meta['etag'] = content_checksum

        data = {'chunks': final_chunks, 'properties': properties or {}}
        # FIXME: we may just pass **obj_meta
        self.container.content_create(account,
                                      container,
                                      obj_name,
                                      size=bytes_transferred,
                                      checksum=content_checksum,
                                      data=data,
                                      stgpol=obj_meta['policy'],
                                      version=obj_meta['version'],
                                      mime_type=obj_meta['mime_type'],
                                      chunk_method=obj_meta['chunk_method'],
                                      **kwargs)
        return final_chunks, bytes_transferred, content_checksum