def create(self, stream):
sysmeta = {}
sysmeta['id'] = self.content_id
sysmeta['version'] = self.version
sysmeta['policy'] = self.stgpol
sysmeta['mime_type'] = self.mime_type
sysmeta['chunk_method'] = self.chunk_method
sysmeta['chunk_size'] = self.metadata['chunk_size']
chunks = _sort_chunks(self.chunks.raw(), self.storage_method.ec)
sysmeta['content_path'] = self.path
sysmeta['container_id'] = self.container_id
headers = {}
handler = ECWriteHandler(stream,
sysmeta,
chunks,
self.storage_method,
headers=headers)
final_chunks, bytes_transferred, content_checksum = handler.stream()
# TODO sanity checks
self.checksum = content_checksum
self._create_object()
return final_chunks, bytes_transferred, content_checksum
def _object_create(self, account, container, obj_name, source,
sysmeta, metadata=None, policy=None, headers=None,
application_key=None):
meta, raw_chunks = self._content_prepare(
account, container, obj_name, sysmeta['content_length'],
policy=policy, headers=headers)
sysmeta['chunk_size'] = int(meta['X-oio-ns-chunk-size'])
sysmeta['id'] = meta[object_headers['id']]
sysmeta['version'] = meta[object_headers['version']]
sysmeta['policy'] = meta[object_headers['policy']]
sysmeta['mime_type'] = meta[object_headers['mime_type']]
sysmeta['chunk_method'] = meta[object_headers['chunk_method']]
storage_method = STORAGE_METHODS.load(sysmeta['chunk_method'])
chunks = _sort_chunks(raw_chunks, storage_method.ec)
sysmeta['content_path'] = obj_name
sysmeta['container_id'] = utils.name2cid(account, container)
sysmeta['ns'] = self.namespace
if storage_method.ec:
handler = ECWriteHandler(source, sysmeta, chunks, storage_method,
headers=headers)
elif storage_method.backblaze:
backblaze_info = self._put_meta_backblaze(storage_method,
application_key)
handler = BackblazeWriteHandler(source, sysmeta,
chunks, storage_method,
headers, backblaze_info)
else:
handler = ReplicatedWriteHandler(source, sysmeta, chunks,
storage_method, headers=headers)
final_chunks, bytes_transferred, content_checksum = handler.stream()
etag = sysmeta['etag']
if etag and etag.lower() != content_checksum.lower():
raise exc.EtagMismatch(
"given etag %s != computed %s" % (etag, content_checksum))
sysmeta['etag'] = content_checksum
h = {}
h[object_headers['size']] = bytes_transferred
h[object_headers['hash']] = sysmeta['etag']
h[object_headers['version']] = sysmeta['version']
h[object_headers['id']] = sysmeta['id']
h[object_headers['policy']] = sysmeta['policy']
h[object_headers['mime_type']] = sysmeta['mime_type']
h[object_headers['chunk_method']] = sysmeta['chunk_method']
if metadata:
for k, v in metadata.iteritems():
h['%sx-%s' % (constants.OBJECT_METADATA_PREFIX, k)] = v
m, body = self._content_create(account, container, obj_name,
final_chunks, headers=h)
return final_chunks, bytes_transferred, content_checksum
def _object_create(self, account, container, obj_name, source,
sysmeta, properties=None, policy=None,
key_file=None, **kwargs):
self._patch_timeouts(kwargs)
obj_meta, chunk_prep = self._content_preparer(
account, container, obj_name,
policy=policy, **kwargs)
obj_meta.update(sysmeta)
obj_meta['content_path'] = obj_name
obj_meta['container_id'] = cid_from_name(account, container).upper()
obj_meta['ns'] = self.namespace
obj_meta['full_path'] = self._generate_fullpath(account, container,
obj_name,
obj_meta['version'])
obj_meta['oio_version'] = (obj_meta.get('oio_version')
or OIO_VERSION)
# XXX content_id is necessary to update an existing object
kwargs['content_id'] = kwargs.get('content_id', obj_meta['id'])
storage_method = STORAGE_METHODS.load(obj_meta['chunk_method'])
if storage_method.ec:
handler = ECWriteHandler(
source, obj_meta, chunk_prep, storage_method, **kwargs)
elif storage_method.backblaze:
backblaze_info = self._b2_credentials(storage_method, key_file)
handler = BackblazeWriteHandler(
source, obj_meta, chunk_prep, storage_method,
backblaze_info, **kwargs)
else:
handler = ReplicatedWriteHandler(
source, obj_meta, chunk_prep, storage_method, **kwargs)
final_chunks, bytes_transferred, content_checksum = handler.stream()
etag = obj_meta.get('etag')
if etag and etag.lower() != content_checksum.lower():
raise exc.EtagMismatch(
"given etag %s != computed %s" % (etag, content_checksum))
obj_meta['etag'] = content_checksum
data = {'chunks': final_chunks, 'properties': properties or {}}
# FIXME: we may just pass **obj_meta
self.container.content_create(
account, container, obj_name, size=bytes_transferred,
checksum=content_checksum, data=data,
stgpol=obj_meta['policy'],
version=obj_meta['version'], mime_type=obj_meta['mime_type'],
chunk_method=obj_meta['chunk_method'],
**kwargs)
return final_chunks, bytes_transferred, content_checksum
def create(self, stream, **kwargs):
sysmeta = self._generate_sysmeta()
chunks = _sort_chunks(self.chunks.raw(), self.storage_method.ec)
headers = {}
handler = ECWriteHandler(
stream, sysmeta, chunks, self.storage_method, headers=headers)
final_chunks, bytes_transferred, content_checksum = handler.stream()
# TODO sanity checks
self.checksum = content_checksum
self._create_object(**kwargs)
return final_chunks, bytes_transferred, content_checksum
def _object_create(
self, account, container, obj_name, source, sysmeta, metadata=None, policy=None, headers=None, key_file=None
):
meta, chunk_prep = self._content_preparer(account, container, obj_name, policy=policy, headers=headers)
sysmeta["chunk_size"] = int(meta["X-oio-ns-chunk-size"])
sysmeta["id"] = meta[object_headers["id"]]
sysmeta["version"] = meta[object_headers["version"]]
sysmeta["policy"] = meta[object_headers["policy"]]
if not sysmeta.get("mime_type"):
sysmeta["mime_type"] = meta[object_headers["mime_type"]]
sysmeta["chunk_method"] = meta[object_headers["chunk_method"]]
sysmeta["content_path"] = obj_name
sysmeta["container_id"] = utils.name2cid(account, container).upper()
sysmeta["ns"] = self.namespace
storage_method = STORAGE_METHODS.load(sysmeta["chunk_method"])
if storage_method.ec:
handler = ECWriteHandler(source, sysmeta, chunk_prep, storage_method, headers=headers)
elif storage_method.backblaze:
backblaze_info = self._b2_credentials(storage_method, key_file)
handler = BackblazeWriteHandler(source, sysmeta, chunk_prep, storage_method, headers, backblaze_info)
else:
handler = ReplicatedWriteHandler(source, sysmeta, chunk_prep, storage_method, headers=headers)
final_chunks, bytes_transferred, content_checksum = handler.stream()
etag = sysmeta["etag"]
if etag and etag.lower() != content_checksum.lower():
raise exc.EtagMismatch("given etag %s != computed %s" % (etag, content_checksum))
sysmeta["etag"] = content_checksum
h = dict()
h.update(headers)
h[object_headers["size"]] = bytes_transferred
h[object_headers["hash"]] = sysmeta["etag"]
h[object_headers["version"]] = sysmeta["version"]
h[object_headers["id"]] = sysmeta["id"]
h[object_headers["policy"]] = sysmeta["policy"]
h[object_headers["mime_type"]] = sysmeta["mime_type"]
h[object_headers["chunk_method"]] = sysmeta["chunk_method"]
m, body = self._content_create(account, container, obj_name, final_chunks, metadata=metadata, headers=h)
return final_chunks, bytes_transferred, content_checksum
def create(self, stream):
sysmeta = {}
sysmeta["id"] = self.content_id
sysmeta["version"] = self.version
sysmeta["policy"] = self.stgpol
sysmeta["mime_type"] = self.mime_type
sysmeta["chunk_method"] = self.chunk_method
sysmeta["chunk_size"] = self.metadata["chunk-size"]
chunks = _sort_chunks(self.chunks.raw(), self.storage_method.ec)
sysmeta["content_path"] = self.path
sysmeta["container_id"] = self.container_id
headers = {}
handler = ECWriteHandler(stream, sysmeta, chunks, self.storage_method, headers=headers)
final_chunks, bytes_transferred, content_checksum = handler.stream()
# TODO sanity checks
self.checksum = content_checksum
self._create_object()
return final_chunks, bytes_transferred, content_checksum
def _object_create(self,
account,
container,
obj_name,
source,
sysmeta,
properties=None,
policy=None,
headers=None,
key_file=None):
obj_meta, chunk_prep = self._content_preparer(account,
container,
obj_name,
policy=policy,
headers=headers)
obj_meta.update(sysmeta)
obj_meta['content_path'] = obj_name
obj_meta['container_id'] = utils.name2cid(account, container).upper()
obj_meta['ns'] = self.namespace
storage_method = STORAGE_METHODS.load(obj_meta['chunk_method'])
if storage_method.ec:
handler = ECWriteHandler(
source,
obj_meta,
chunk_prep,
storage_method,
headers=headers,
write_timeout=self.write_timeout,
read_timeout=self.read_timeout,
connection_timeout=self.connection_timeout)
elif storage_method.backblaze:
backblaze_info = self._b2_credentials(storage_method, key_file)
handler = BackblazeWriteHandler(source, obj_meta, chunk_prep,
storage_method, headers,
backblaze_info)
else:
handler = ReplicatedWriteHandler(
source,
obj_meta,
chunk_prep,
storage_method,
headers=headers,
write_timeout=self.write_timeout,
read_timeout=self.read_timeout,
connection_timeout=self.connection_timeout)
final_chunks, bytes_transferred, content_checksum = handler.stream()
etag = obj_meta.get('etag')
if etag and etag.lower() != content_checksum.lower():
raise exc.EtagMismatch("given etag %s != computed %s" %
(etag, content_checksum))
obj_meta['etag'] = content_checksum
data = {'chunks': final_chunks, 'properties': properties or {}}
# FIXME: we may just pass **obj_meta
self.container.content_create(account,
container,
obj_name,
size=bytes_transferred,
checksum=content_checksum,
data=data,
content_id=obj_meta['id'],
stgpol=obj_meta['policy'],
version=obj_meta['version'],
mime_type=obj_meta['mime_type'],
chunk_method=obj_meta['chunk_method'],
headers=headers)
return final_chunks, bytes_transferred, content_checksum