def object_fetch(self,
account,
container,
obj,
version=None,
ranges=None,
key_file=None,
**kwargs):
meta, raw_chunks = self.object_locate(account,
container,
obj,
version=version,
**kwargs)
chunk_method = meta['chunk_method']
storage_method = STORAGE_METHODS.load(chunk_method)
chunks = _sort_chunks(raw_chunks, storage_method.ec)
meta['container_id'] = name2cid(account, container).upper()
meta['ns'] = self.namespace
self._patch_timeouts(kwargs)
if storage_method.ec:
stream = fetch_stream_ec(chunks, ranges, storage_method, **kwargs)
elif storage_method.backblaze:
stream = self._fetch_stream_backblaze(meta, chunks, ranges,
storage_method, key_file,
**kwargs)
else:
stream = fetch_stream(chunks, ranges, storage_method, **kwargs)
return meta, stream
def object_fetch(self,
account,
container,
obj,
ranges=None,
headers=None,
key_file=None):
if not headers:
headers = dict()
if 'X-oio-req-id' not in headers:
headers['X-oio-req-id'] = utils.request_id()
meta, raw_chunks = self.object_locate(account,
container,
obj,
headers=headers)
chunk_method = meta['chunk_method']
storage_method = STORAGE_METHODS.load(chunk_method)
chunks = _sort_chunks(raw_chunks, storage_method.ec)
meta['container_id'] = utils.name2cid(account, container).upper()
meta['ns'] = self.namespace
if storage_method.ec:
stream = self._fetch_stream_ec(meta, chunks, ranges,
storage_method, headers)
elif storage_method.backblaze:
stream = self._fetch_stream_backblaze(meta, chunks, ranges,
storage_method, key_file)
else:
stream = self._fetch_stream(meta, chunks, ranges, storage_method,
headers)
return meta, stream
def _object_create(self, account, container, obj_name, source,
sysmeta, metadata=None, policy=None, headers=None,
application_key=None):
meta, raw_chunks = self._content_prepare(
account, container, obj_name, sysmeta['content_length'],
policy=policy, headers=headers)
sysmeta['chunk_size'] = int(meta['X-oio-ns-chunk-size'])
sysmeta['id'] = meta[object_headers['id']]
sysmeta['version'] = meta[object_headers['version']]
sysmeta['policy'] = meta[object_headers['policy']]
sysmeta['mime_type'] = meta[object_headers['mime_type']]
sysmeta['chunk_method'] = meta[object_headers['chunk_method']]
storage_method = STORAGE_METHODS.load(sysmeta['chunk_method'])
chunks = _sort_chunks(raw_chunks, storage_method.ec)
sysmeta['content_path'] = obj_name
sysmeta['container_id'] = utils.name2cid(account, container)
sysmeta['ns'] = self.namespace
if storage_method.ec:
handler = ECWriteHandler(source, sysmeta, chunks, storage_method,
headers=headers)
elif storage_method.backblaze:
backblaze_info = self._put_meta_backblaze(storage_method,
application_key)
handler = BackblazeWriteHandler(source, sysmeta,
chunks, storage_method,
headers, backblaze_info)
else:
handler = ReplicatedWriteHandler(source, sysmeta, chunks,
storage_method, headers=headers)
final_chunks, bytes_transferred, content_checksum = handler.stream()
etag = sysmeta['etag']
if etag and etag.lower() != content_checksum.lower():
raise exc.EtagMismatch(
"given etag %s != computed %s" % (etag, content_checksum))
sysmeta['etag'] = content_checksum
h = {}
h[object_headers['size']] = bytes_transferred
h[object_headers['hash']] = sysmeta['etag']
h[object_headers['version']] = sysmeta['version']
h[object_headers['id']] = sysmeta['id']
h[object_headers['policy']] = sysmeta['policy']
h[object_headers['mime_type']] = sysmeta['mime_type']
h[object_headers['chunk_method']] = sysmeta['chunk_method']
if metadata:
for k, v in metadata.iteritems():
h['%sx-%s' % (constants.OBJECT_METADATA_PREFIX, k)] = v
m, body = self._content_create(account, container, obj_name,
final_chunks, headers=h)
return final_chunks, bytes_transferred, content_checksum
def object_fetch(self, account, container, obj, ranges=None, headers=None, key_file=None):
if not headers:
headers = dict()
if "X-oio-req-id" not in headers:
headers["X-oio-req-id"] = utils.request_id()
meta, raw_chunks = self.object_analyze(account, container, obj, headers=headers)
chunk_method = meta["chunk-method"]
storage_method = STORAGE_METHODS.load(chunk_method)
chunks = _sort_chunks(raw_chunks, storage_method.ec)
meta["container_id"] = utils.name2cid(account, container).upper()
meta["ns"] = self.namespace
if storage_method.ec:
stream = self._fetch_stream_ec(meta, chunks, ranges, storage_method, headers)
elif storage_method.backblaze:
stream = self._fetch_stream_backblaze(meta, chunks, ranges, storage_method, key_file)
else:
stream = self._fetch_stream(meta, chunks, ranges, storage_method, headers)
return meta, stream
def object_delete(self, account, container, obj, headers={},
application_key=None):
uri = self._make_uri('content/delete')
params = self._make_params(account, container, obj)
meta, raw_chunks = self.object_analyze(
account, container, obj, headers=headers)
if meta:
chunk_method = meta['chunk-method']
storage_method = STORAGE_METHODS.load(chunk_method)
meta['ns'] = self.namespace
meta['container_id'] = utils.name2cid(account, container)
chunks = _sort_chunks(raw_chunks, storage_method.ec)
if storage_method.backblaze:
backblaze_info = self._put_meta_backblaze(storage_method,
application_key)
BackblazeDeleteHandler(meta, chunks, backblaze_info).delete()
resp, resp_body = self._request(
'POST', uri, params=params, headers=headers)
def object_fetch(self, account, container, obj, ranges=None,
headers=None, key_file=None):
meta, raw_chunks = self.object_analyze(
account, container, obj, headers=headers)
chunk_method = meta['chunk-method']
storage_method = STORAGE_METHODS.load(chunk_method)
chunks = _sort_chunks(raw_chunks, storage_method.ec)
meta['container_id'] = utils.name2cid(account, container).upper()
meta['ns'] = self.namespace
if storage_method.ec:
stream = self._fetch_stream_ec(meta, chunks, ranges,
storage_method, headers)
elif storage_method.backblaze:
stream = self._fetch_stream_backblaze(meta, chunks, ranges,
storage_method, key_file)
else:
stream = self._fetch_stream(meta, chunks, ranges, storage_method,
headers)
return meta, stream
def _object_create(
self, account, container, obj_name, source, sysmeta, metadata=None, policy=None, headers=None, key_file=None
):
meta, chunk_prep = self._content_preparer(account, container, obj_name, policy=policy, headers=headers)
sysmeta["chunk_size"] = int(meta["X-oio-ns-chunk-size"])
sysmeta["id"] = meta[object_headers["id"]]
sysmeta["version"] = meta[object_headers["version"]]
sysmeta["policy"] = meta[object_headers["policy"]]
if not sysmeta.get("mime_type"):
sysmeta["mime_type"] = meta[object_headers["mime_type"]]
sysmeta["chunk_method"] = meta[object_headers["chunk_method"]]
sysmeta["content_path"] = obj_name
sysmeta["container_id"] = utils.name2cid(account, container).upper()
sysmeta["ns"] = self.namespace
storage_method = STORAGE_METHODS.load(sysmeta["chunk_method"])
if storage_method.ec:
handler = ECWriteHandler(source, sysmeta, chunk_prep, storage_method, headers=headers)
elif storage_method.backblaze:
backblaze_info = self._b2_credentials(storage_method, key_file)
handler = BackblazeWriteHandler(source, sysmeta, chunk_prep, storage_method, headers, backblaze_info)
else:
handler = ReplicatedWriteHandler(source, sysmeta, chunk_prep, storage_method, headers=headers)
final_chunks, bytes_transferred, content_checksum = handler.stream()
etag = sysmeta["etag"]
if etag and etag.lower() != content_checksum.lower():
raise exc.EtagMismatch("given etag %s != computed %s" % (etag, content_checksum))
sysmeta["etag"] = content_checksum
h = dict()
h.update(headers)
h[object_headers["size"]] = bytes_transferred
h[object_headers["hash"]] = sysmeta["etag"]
h[object_headers["version"]] = sysmeta["version"]
h[object_headers["id"]] = sysmeta["id"]
h[object_headers["policy"]] = sysmeta["policy"]
h[object_headers["mime_type"]] = sysmeta["mime_type"]
h[object_headers["chunk_method"]] = sysmeta["chunk_method"]
m, body = self._content_create(account, container, obj_name, final_chunks, metadata=metadata, headers=h)
return final_chunks, bytes_transferred, content_checksum
def object_fetch(self, account, container, obj, ranges=None,
headers=None, application_key=None):
meta, raw_chunks = self.object_analyze(
account, container, obj, headers=headers)
chunk_method = meta['chunk-method']
storage_method = STORAGE_METHODS.load(chunk_method)
chunks = _sort_chunks(raw_chunks, storage_method.ec)
meta['container_id'] = utils.name2cid(account, container)
meta['ns'] = self.namespace
if storage_method.ec:
stream = self._fetch_stream_ec(meta, chunks, ranges,
storage_method, headers)
elif storage_method.backblaze:
stream = self._fetch_stream_backblaze(meta, chunks, ranges,
storage_method,
application_key)
else:
stream = self._fetch_stream(meta, chunks, ranges, storage_method,
headers)
return meta, stream
def _object_create(self,
account,
container,
obj_name,
source,
sysmeta,
properties=None,
policy=None,
headers=None,
key_file=None):
obj_meta, chunk_prep = self._content_preparer(account,
container,
obj_name,
policy=policy,
headers=headers)
obj_meta.update(sysmeta)
obj_meta['content_path'] = obj_name
obj_meta['container_id'] = utils.name2cid(account, container).upper()
obj_meta['ns'] = self.namespace
storage_method = STORAGE_METHODS.load(obj_meta['chunk_method'])
if storage_method.ec:
handler = ECWriteHandler(
source,
obj_meta,
chunk_prep,
storage_method,
headers=headers,
write_timeout=self.write_timeout,
read_timeout=self.read_timeout,
connection_timeout=self.connection_timeout)
elif storage_method.backblaze:
backblaze_info = self._b2_credentials(storage_method, key_file)
handler = BackblazeWriteHandler(source, obj_meta, chunk_prep,
storage_method, headers,
backblaze_info)
else:
handler = ReplicatedWriteHandler(
source,
obj_meta,
chunk_prep,
storage_method,
headers=headers,
write_timeout=self.write_timeout,
read_timeout=self.read_timeout,
connection_timeout=self.connection_timeout)
final_chunks, bytes_transferred, content_checksum = handler.stream()
etag = obj_meta.get('etag')
if etag and etag.lower() != content_checksum.lower():
raise exc.EtagMismatch("given etag %s != computed %s" %
(etag, content_checksum))
obj_meta['etag'] = content_checksum
data = {'chunks': final_chunks, 'properties': properties or {}}
# FIXME: we may just pass **obj_meta
self.container.content_create(account,
container,
obj_name,
size=bytes_transferred,
checksum=content_checksum,
data=data,
content_id=obj_meta['id'],
stgpol=obj_meta['policy'],
version=obj_meta['version'],
mime_type=obj_meta['mime_type'],
chunk_method=obj_meta['chunk_method'],
headers=headers)
return final_chunks, bytes_transferred, content_checksum
def _object_create(self,
account,
container,
obj_name,
source,
sysmeta,
properties=None,
policy=None,
key_file=None,
**kwargs):
self._patch_timeouts(kwargs)
obj_meta, chunk_prep = self._content_preparer(account,
container,
obj_name,
policy=policy,
**kwargs)
obj_meta.update(sysmeta)
obj_meta['content_path'] = obj_name
obj_meta['container_id'] = name2cid(account, container).upper()
obj_meta['ns'] = self.namespace
obj_meta['full_path'] = self._generate_fullpath(
account, container, obj_name, obj_meta['version'])
obj_meta['oio_version'] = (obj_meta.get('oio_version') or OIO_VERSION)
# XXX content_id is necessary to update an existing object
kwargs['content_id'] = kwargs.get('content_id', obj_meta['id'])
storage_method = STORAGE_METHODS.load(obj_meta['chunk_method'])
if storage_method.ec:
handler = ECWriteHandler(source, obj_meta, chunk_prep,
storage_method, **kwargs)
elif storage_method.backblaze:
backblaze_info = self._b2_credentials(storage_method, key_file)
handler = BackblazeWriteHandler(source, obj_meta, chunk_prep,
storage_method, backblaze_info,
**kwargs)
else:
handler = ReplicatedWriteHandler(source, obj_meta, chunk_prep,
storage_method, **kwargs)
final_chunks, bytes_transferred, content_checksum = handler.stream()
etag = obj_meta.get('etag')
if etag and etag.lower() != content_checksum.lower():
raise exc.EtagMismatch("given etag %s != computed %s" %
(etag, content_checksum))
obj_meta['etag'] = content_checksum
data = {'chunks': final_chunks, 'properties': properties or {}}
# FIXME: we may just pass **obj_meta
self.container.content_create(account,
container,
obj_name,
size=bytes_transferred,
checksum=content_checksum,
data=data,
stgpol=obj_meta['policy'],
version=obj_meta['version'],
mime_type=obj_meta['mime_type'],
chunk_method=obj_meta['chunk_method'],
**kwargs)
return final_chunks, bytes_transferred, content_checksum
