class ObjectStorageApi(object):
"""
The Object Storage API.
High level API that wraps `AccountClient`, `ContainerClient` and
`DirectoryClient` classes.
Every method that takes a `kwargs` argument accepts the at least
the following keywords:
- `headers`: `dict` of extra headers to pass to the proxy
- `connection_timeout`: `float`
- `read_timeout`: `float`
- `write_timeout`: `float`
"""
TIMEOUT_KEYS = ('connection_timeout', 'read_timeout', 'write_timeout')
def __init__(self, namespace, logger=None, **kwargs):
"""
Initialize the object storage API.
:param namespace: name of the namespace to interract with
:type namespace: `str`
:keyword connection_timeout: connection timeout towards rawx services
:type connection_timeout: `float` seconds
:keyword read_timeout: timeout for rawx responses and data reads from
the caller (when uploading)
:type read_timeout: `float` seconds
:keyword write_timeout: timeout for rawx write requests
:type write_timeout: `float` seconds
:keyword pool_manager: a pooled connection manager that will be used
for all HTTP based APIs (except rawx)
:type pool_manager: `urllib3.PoolManager`
"""
self.namespace = namespace
conf = {"namespace": self.namespace}
self.logger = logger or get_logger(conf)
self.timeouts = {tok: float_value(tov, None)
for tok, tov in kwargs.items()
if tok in self.__class__.TIMEOUT_KEYS}
from oio.account.client import AccountClient
from oio.container.client import ContainerClient
from oio.directory.client import DirectoryClient
self.directory = DirectoryClient(conf, logger=self.logger, **kwargs)
self.container = ContainerClient(conf, logger=self.logger, **kwargs)
# In AccountClient, "endpoint" is the account service, not the proxy
acct_kwargs = kwargs.copy()
acct_kwargs["proxy_endpoint"] = acct_kwargs.pop("endpoint", None)
self.account = AccountClient(conf, logger=self.logger, **acct_kwargs)
def _patch_timeouts(self, kwargs):
"""
Insert timeout settings from this class's constructor into `kwargs`,
if they are not already there.
"""
for tok, tov in self.timeouts.items():
if tok not in kwargs:
kwargs[tok] = tov
def account_create(self, account, **kwargs):
"""
Create an account.
:param account: name of the account to create
:type account: `str`
:returns: `True` if the account has been created
"""
return self.account.account_create(account, **kwargs)
@handle_account_not_found
def account_delete(self, account, **kwargs):
"""
Delete an account.
:param account: name of the account to delete
:type account: `str`
"""
self.account.account_delete(account, **kwargs)
@handle_account_not_found
def account_show(self, account, **kwargs):
"""
Get information about an account.
"""
return self.account.account_show(account, **kwargs)
def account_list(self, **kwargs):
"""
List known accounts.
Notice that account creation is asynchronous, and an autocreated
account may appear in the listing only after several seconds.
"""
return self.account.account_list(**kwargs)
@handle_account_not_found
def account_update(self, account, metadata, to_delete=None, **kwargs):
warnings.warn("You'd better use account_set_properties()",
DeprecationWarning, stacklevel=2)
self.account.account_update(account, metadata, to_delete, **kwargs)
@handle_account_not_found
def account_set_properties(self, account, properties, **kwargs):
self.account.account_update(account, properties, None, **kwargs)
@handle_account_not_found
def account_del_properties(self, account, properties, **kwargs):
self.account.account_update(account, None, properties, **kwargs)
def container_create(self, account, container, properties=None,
**kwargs):
"""
Create a container.
:param account: account in which to create the container
:type account: `str`
:param container: name of the container
:type container: `str`
:param properties: properties to set on the container
:type properties: `dict`
:returns: True if the container has been created,
False if it already exists
"""
return self.container.container_create(account, container,
properties=properties,
**kwargs)
@handle_container_not_found
@ensure_headers
@ensure_request_id
def container_touch(self, account, container, **kwargs):
"""
Trigger a notification about the container state.
:param account: account from which to delete the container
:type account: `str`
:param container: name of the container
:type container: `str`
"""
self.container.container_touch(account, container, **kwargs)
def container_create_many(self, account, containers, properties=None,
**kwargs):
"""
Create Many containers
:param account: account in which to create the containers
:type account: `str`
:param containers: names of the containers
:type containers: `list`
:param properties: properties to set on the containers
:type properties: `dict`
"""
return self.container.container_create_many(account,
containers,
properties=properties,
**kwargs)
@handle_container_not_found
def container_delete(self, account, container, **kwargs):
"""
Delete a container.
:param account: account from which to delete the container
:type account: `str`
:param container: name of the container
:type container: `str`
"""
self.container.container_delete(account, container, **kwargs)
@handle_account_not_found
def container_list(self, account, limit=None, marker=None,
end_marker=None, prefix=None, delimiter=None,
**kwargs):
"""
Get the list of containers of an account.
:param account: account from which to get the container list
:type account: `str`
:keyword limit: maximum number of results to return
:type limit: `int`
:keyword marker: name of the container from where to start the listing
:type marker: `str`
:keyword end_marker:
:keyword prefix:
:keyword delimiter:
:return: the list of containers of an account
:rtype: `list` of items (`list`) with 4 fields:
name, number of objects, number of bytes, and 1 if the item
is a prefix or 0 if the item is actually a container
"""
resp = self.account.container_list(account, limit=limit,
marker=marker,
end_marker=end_marker,
prefix=prefix,
delimiter=delimiter,
**kwargs)
return resp["listing"]
@handle_container_not_found
def container_show(self, account, container, **kwargs):
"""
Get information about a container (user properties).
:param account: account in which the container is
:type account: `str`
:param container: name of the container
:type container: `str`
:returns: a `dict` with "properties" containing a `dict`
of user properties.
"""
return self.container.container_show(account, container, **kwargs)
@handle_container_not_found
def container_snapshot(self, account, container, dst_account,
dst_container, batch=100, **kwargs):
"""
Create a copy of the container (only the content of the database)
:param account: account in which the target is
:type account: `str`
:param container: name of the target
:type container: `str`
:param dst_account: account in which the snapshot will be.
:type dst_account: `str`
:param dst_container: name of the snapshot
:type dst_container: `str`
"""
try:
self.container.container_freeze(account, container)
self.container.container_snapshot(
account, container, dst_account, dst_container)
resp = self.object_list(dst_account, dst_container)
obj_gen = resp['objects']
target_beans = []
copy_beans = []
for obj in obj_gen:
data = self.object_locate(
account, container, obj["name"])
chunks = [chunk['url'] for chunk in data[1]]
copies = self._generate_copy(chunks)
fullpath = self._generate_fullpath(
dst_account, dst_container, obj['name'], obj['version'])
self._send_copy(chunks, copies, fullpath[0])
t_beans, c_beans = self._prepare_update_meta2(
data[1], copies, dst_account, dst_container,
obj['content'])
target_beans.extend(t_beans)
copy_beans.extend(c_beans)
if len(target_beans) > batch:
self.container.container_raw_update(
target_beans, copy_beans,
dst_account, dst_container,
frozen=True)
target_beans = []
copy_beans = []
if target_beans:
self.container.container_raw_update(
target_beans, copy_beans,
dst_account, dst_container,
frozen=True)
finally:
self.container.container_enable(account, container)
@handle_container_not_found
def container_get_properties(self, account, container, properties=None,
**kwargs):
"""
Get information about a container (user and system properties).
:param account: account in which the container is
:type account: `str`
:param container: name of the container
:type container: `str`
:param properties: *ignored*
:returns: a `dict` with "properties" and "system" entries,
containing respectively a `dict` of user properties and
a `dict` of system properties.
"""
return self.container.container_get_properties(account, container,
properties=properties,
**kwargs)
@handle_container_not_found
def container_set_properties(self, account, container, properties=None,
clear=False, **kwargs):
"""
Set properties on a container.
:param account: name of the account
:type account: `str`
:param container: name of the container where to set properties
:type container: `str`
:param properties: a dictionary of properties
:type properties: `dict`
:param clear:
:type clear: `bool`
:keyword system: dictionary of system properties to set
"""
return self.container.container_set_properties(
account, container, properties,
clear=clear, **kwargs)
@handle_container_not_found
def container_del_properties(self, account, container, properties,
**kwargs):
"""
Delete properties of a container.
:param account: name of the account
:type account: `str`
:param container: name of the container to deal with
:type container: `str`
:param properties: a list of property keys
:type properties: `list`
"""
return self.container.container_del_properties(
account, container, properties, **kwargs)
def container_update(self, account, container, metadata, clear=False,
**kwargs):
warnings.warn("You'd better use container_set_properties()",
DeprecationWarning)
if not metadata:
self.container_del_properties(
account, container, [], **kwargs)
else:
self.container_set_properties(
account, container, metadata, clear, **kwargs)
@handle_container_not_found
@ensure_headers
@ensure_request_id
def object_create(self, account, container, file_or_path=None, data=None,
etag=None, obj_name=None, mime_type=None,
metadata=None, policy=None, key_file=None,
append=False, properties=None, **kwargs):
"""
Create an object or append data to object in *container* of *account*
with data taken from either *data* (`str` or `generator`) or
*file_or_path* (path to a file or file-like object).
The object will be named after *obj_name* if specified, or after
the base name of *file_or_path*.
:param account: name of the account where to create the object
:type account: `str`
:param container: name of the container where to create the object
:type container: `str`
:param file_or_path: file-like object or path to a file from which
to read object data
:type file_or_path: `str` or file-like object
:param data: object data (if `file_or_path` is not set)
:type data: `str` or `generator`
:keyword etag: entity tag of the object
:type etag: `str`
:keyword obj_name: name of the object to create. If not set, will use
the base name of `file_or_path`.
:keyword mime_type: MIME type of the object
:type mime_type: `str`
:keyword properties: a dictionary of properties
:type properties: `dict`
:keyword policy: name of the storage policy
:type policy: `str`
:keyword key_file:
:param append: if set, data will be append to existing object (or
object will be created if unset)
:type append: `bool`
:returns: `list` of chunks, size and hash of the what has been uploaded
"""
if (data, file_or_path) == (None, None):
raise exc.MissingData()
src = data if data is not None else file_or_path
if src is file_or_path:
if isinstance(file_or_path, basestring):
if not os.path.exists(file_or_path):
raise exc.FileNotFound("File '%s' not found." %
file_or_path)
file_name = os.path.basename(file_or_path)
else:
try:
file_name = os.path.basename(file_or_path.name)
except AttributeError:
file_name = None
obj_name = obj_name or file_name
elif isgenerator(src):
file_or_path = GeneratorIO(src)
src = file_or_path
if not obj_name:
raise exc.MissingName(
"No name for the object has been specified"
)
sysmeta = {'mime_type': mime_type,
'etag': etag}
if metadata:
warnings.warn(
"You'd better use 'properties' instead of 'metadata'",
DeprecationWarning, stacklevel=4)
if not properties:
properties = metadata
else:
properties.update(metadata)
if src is data:
return self._object_create(
account, container, obj_name, BytesIO(data), sysmeta,
properties=properties, policy=policy,
key_file=key_file, append=append, **kwargs)
elif hasattr(file_or_path, "read"):
return self._object_create(
account, container, obj_name, src, sysmeta,
properties=properties, policy=policy, key_file=key_file,
append=append, **kwargs)
else:
with open(file_or_path, "rb") as f:
return self._object_create(
account, container, obj_name, f, sysmeta,
properties=properties, policy=policy,
key_file=key_file, append=append, **kwargs)
@ensure_headers
@ensure_request_id
def object_touch(self, account, container, obj,
version=None, **kwargs):
"""
Trigger a notification about an object
(as if it just had been created).
:param account: name of the account where to create the object
:type account: `str`
:param container: name of the container where to create the object
:type container: `str`
:param obj: name of the object to touch
"""
self.container.content_touch(account, container, obj,
version=version, **kwargs)
def object_drain(self, account, container, obj,
version=None, **kwargs):
"""
Remove all the chunks of a content, but keep all the metadata.
:param account: name of the account where the object is present
:type account: `str`
:param container: name of the container where the object is present
:type container: `str`
:param obj: name of the object to drain
"""
self.container.content_drain(account, container, obj,
version=version, **kwargs)
@handle_object_not_found
@ensure_headers
@ensure_request_id
def object_delete(self, account, container, obj,
version=None, **kwargs):
"""
Delete an object from a container. If versioning is enabled and no
version is specified, the object will be marked as deleted but not
actually deleted.
:param account: name of the account the object belongs to
:type account: `str`
:param container: name of the container the object belongs to
:type container: `str`
:param obj: name of the object to delete
:param version: version of the object to delete
:returns: True on success
"""
return self.container.content_delete(account, container, obj,
version=version, **kwargs)
@ensure_headers
@ensure_request_id
def object_delete_many(self, account, container, objs, **kwargs):
return self.container.content_delete_many(
account, container, objs, **kwargs)
@handle_object_not_found
@ensure_headers
@ensure_request_id
def object_truncate(self, account, container, obj,
version=None, size=None, **kwargs):
"""
Truncate object at specified size. Only shrink is supported.
A download may occur if size is not on chunk boundaries.
:param account: name of the account in which the object is stored
:param container: name of the container in which the object is stored
:param obj: name of the object to query
:param version: version of the object to query
:param size: new size of object
"""
# code copied from object_fetch (should be factorized !)
meta, raw_chunks = self.object_locate(
account, container, obj, version=version, **kwargs)
chunk_method = meta['chunk_method']
storage_method = STORAGE_METHODS.load(chunk_method)
chunks = _sort_chunks(raw_chunks, storage_method.ec)
for pos in sorted(chunks.keys()):
chunk = chunks[pos][0]
if (size >= chunk['offset']
and size <= chunk['offset'] + chunk['size']):
break
else:
raise exc.OioException("No chunk found at position %d" % size)
if chunk['offset'] != size:
# retrieve partial chunk
ret = self.object_fetch(account, container, obj,
version=version,
ranges=[(chunk['offset'], size-1)])
# TODO implement a proper object_update
pos = int(chunk['pos'].split('.')[0])
self.object_create(account, container, obj_name=obj,
data=ret[1], meta_pos=pos,
content_id=meta['id'])
return self.container.content_truncate(account, container, obj,
version=version, size=size,
**kwargs)
@handle_container_not_found
def object_list(self, account, container, limit=None, marker=None,
delimiter=None, prefix=None, end_marker=None,
properties=False, versions=False, deleted=False,
**kwargs):
"""
Lists objects inside a container.
:param properties: if True, list object properties along with objects
:param versions: if True, list all versions of objects
:param deleted: if True, list also the deleted objects
:returns: a dict which contains
* 'objects': the list of objects
* 'prefixes': common prefixes (only if delimiter and prefix are set)
* 'properties': a dict of container properties
* 'system': a dict of system metadata
"""
_, resp_body = self.container.content_list(
account, container, limit=limit, marker=marker,
end_marker=end_marker, prefix=prefix, delimiter=delimiter,
properties=properties, versions=versions, deleted=deleted,
**kwargs)
for obj in resp_body['objects']:
mtype = obj.get('mime-type')
if mtype is not None:
obj['mime_type'] = mtype
del obj['mime-type']
version = obj.get('ver')
if version is not None:
obj['version'] = version
del obj['ver']
return resp_body
@handle_object_not_found
def object_locate(self, account, container, obj,
version=None, **kwargs):
"""
Get a description of the object along with the list of its chunks.
:param account: name of the account in which the object is stored
:param container: name of the container in which the object is stored
:param obj: name of the object to query
:param version: version of the object to query
:returns: a tuple with object metadata `dict` as first element
and chunk `list` as second element
"""
obj_meta, chunks = self.container.content_locate(
account, container, obj, version=version, **kwargs)
return obj_meta, chunks
def object_analyze(self, *args, **kwargs):
"""
:deprecated: use `object_locate`
"""
warnings.warn("You'd better use object_locate()",
DeprecationWarning)
return self.object_locate(*args, **kwargs)
@ensure_headers
@ensure_request_id
def object_fetch(self, account, container, obj, version=None, ranges=None,
key_file=None, **kwargs):
meta, raw_chunks = self.object_locate(
account, container, obj, version=version, **kwargs)
chunk_method = meta['chunk_method']
storage_method = STORAGE_METHODS.load(chunk_method)
chunks = _sort_chunks(raw_chunks, storage_method.ec)
meta['container_id'] = cid_from_name(account, container).upper()
meta['ns'] = self.namespace
self._patch_timeouts(kwargs)
if storage_method.ec:
stream = fetch_stream_ec(chunks, ranges, storage_method, **kwargs)
elif storage_method.backblaze:
stream = self._fetch_stream_backblaze(meta, chunks, ranges,
storage_method, key_file,
**kwargs)
else:
stream = fetch_stream(chunks, ranges, storage_method, **kwargs)
return meta, stream
@handle_object_not_found
def object_get_properties(self, account, container, obj, **kwargs):
return self.container.content_get_properties(account, container, obj,
**kwargs)
@handle_object_not_found
def object_show(self, account, container, obj, version=None, **kwargs):
"""
Get a description of the content along with its user properties.
:param account: name of the account in which the object is stored
:param container: name of the container in which the object is stored
:param obj: name of the object to query
:returns: a `dict` describing the object
.. python::
{'hash': '6BF60C17CC15EEA108024903B481738F',
'ctime': '1481031763',
'deleted': 'False',
'properties': {
u'projet': u'OpenIO-SDS'},
'length': '43518',
'hash_method': 'md5',
'chunk_method': 'ec/algo=liberasurecode_rs_vand,k=6,m=3',
'version': '1481031762951972',
'policy': 'EC',
'id': '20BF2194FD420500CD4729AE0B5CBC07',
'mime_type': 'application/octet-stream',
'name': 'Makefile'}
"""
return self.container.content_show(account, container, obj,
version=version,
**kwargs)
def object_update(self, account, container, obj, metadata,
version=None, clear=False, **kwargs):
warnings.warn("You'd better use object_set_properties()",
DeprecationWarning, stacklevel=2)
if clear:
self.object_del_properties(
account, container, obj, [], version=version, **kwargs)
if metadata:
self.object_set_properties(
account, container, obj, metadata, version=version, **kwargs)
@handle_object_not_found
def object_set_properties(self, account, container, obj, properties,
version=None, **kwargs):
return self.container.content_set_properties(
account, container, obj, properties={'properties': properties},
version=version, **kwargs)
@handle_object_not_found
def object_del_properties(self, account, container, obj, properties,
version=None, **kwargs):
return self.container.content_del_properties(
account, container, obj, properties=properties,
version=version, **kwargs)
def _content_preparer(self, account, container, obj_name,
policy=None, **kwargs):
# TODO: optimize by asking more than one metachunk at a time
obj_meta, first_body = self.container.content_prepare(
account, container, obj_name, size=1, stgpol=policy,
autocreate=True, **kwargs)
storage_method = STORAGE_METHODS.load(obj_meta['chunk_method'])
def _fix_mc_pos(chunks, mc_pos):
for chunk in chunks:
raw_pos = chunk["pos"].split(".")
if storage_method.ec:
chunk['num'] = int(raw_pos[1])
chunk["pos"] = "%d.%d" % (mc_pos, chunk['num'])
else:
chunk["pos"] = str(mc_pos)
def _metachunk_preparer():
mc_pos = kwargs.get('meta_pos', 0)
_fix_mc_pos(first_body, mc_pos)
yield first_body
while True:
mc_pos += 1
_, next_body = self.container.content_prepare(
account, container, obj_name, 1, stgpol=policy,
autocreate=True, **kwargs)
_fix_mc_pos(next_body, mc_pos)
yield next_body
return obj_meta, _metachunk_preparer
def _generate_fullpath(self, account, container_name, path, version):
return ['{0}/{1}/{2}/{3}'.format(quote_plus(account),
quote_plus(container_name),
quote_plus(path),
version)]
def _object_create(self, account, container, obj_name, source,
sysmeta, properties=None, policy=None,
key_file=None, **kwargs):
self._patch_timeouts(kwargs)
obj_meta, chunk_prep = self._content_preparer(
account, container, obj_name,
policy=policy, **kwargs)
obj_meta.update(sysmeta)
obj_meta['content_path'] = obj_name
obj_meta['container_id'] = cid_from_name(account, container).upper()
obj_meta['ns'] = self.namespace
obj_meta['full_path'] = self._generate_fullpath(account, container,
obj_name,
obj_meta['version'])
obj_meta['oio_version'] = (obj_meta.get('oio_version')
or OIO_VERSION)
# XXX content_id is necessary to update an existing object
kwargs['content_id'] = kwargs.get('content_id', obj_meta['id'])
storage_method = STORAGE_METHODS.load(obj_meta['chunk_method'])
if storage_method.ec:
handler = ECWriteHandler(
source, obj_meta, chunk_prep, storage_method, **kwargs)
elif storage_method.backblaze:
backblaze_info = self._b2_credentials(storage_method, key_file)
handler = BackblazeWriteHandler(
source, obj_meta, chunk_prep, storage_method,
backblaze_info, **kwargs)
else:
handler = ReplicatedWriteHandler(
source, obj_meta, chunk_prep, storage_method, **kwargs)
final_chunks, bytes_transferred, content_checksum = handler.stream()
etag = obj_meta.get('etag')
if etag and etag.lower() != content_checksum.lower():
raise exc.EtagMismatch(
"given etag %s != computed %s" % (etag, content_checksum))
obj_meta['etag'] = content_checksum
data = {'chunks': final_chunks, 'properties': properties or {}}
# FIXME: we may just pass **obj_meta
self.container.content_create(
account, container, obj_name, size=bytes_transferred,
checksum=content_checksum, data=data,
stgpol=obj_meta['policy'],
version=obj_meta['version'], mime_type=obj_meta['mime_type'],
chunk_method=obj_meta['chunk_method'],
**kwargs)
return final_chunks, bytes_transferred, content_checksum
def _b2_credentials(self, storage_method, key_file):
key_file = key_file or '/etc/oio/sds/b2-appkey.conf'
try:
return BackblazeUtils.get_credentials(storage_method, key_file)
except BackblazeUtilsException as err:
raise exc.ConfigurationException(str(err))
def _fetch_stream_backblaze(self, meta, chunks, ranges,
storage_method, key_file,
**kwargs):
backblaze_info = self._b2_credentials(storage_method, key_file)
total_bytes = 0
current_offset = 0
size = None
offset = 0
for pos in range(len(chunks)):
if ranges:
offset = ranges[pos][0]
size = ranges[pos][1]
if size is None:
size = int(meta["length"])
chunk_size = int(chunks[pos][0]["size"])
if total_bytes >= size:
break
if current_offset + chunk_size > offset:
if current_offset < offset:
_offset = offset - current_offset
else:
_offset = 0
if chunk_size + total_bytes > size:
_size = size - total_bytes
else:
_size = chunk_size
handler = BackblazeChunkDownloadHandler(
meta, chunks[pos], _offset, _size,
backblaze_info=backblaze_info)
stream = handler.get_stream()
if not stream:
raise exc.OioException("Error while downloading")
total_bytes += len(stream)
yield stream
current_offset += chunk_size
@handle_container_not_found
def container_refresh(self, account, container, attempts=3, **kwargs):
for i in range(attempts):
try:
self.account.container_reset(account, container, time.time())
except exc.Conflict:
if i >= attempts - 1:
raise
try:
self.container.container_touch(account, container)
except exc.ClientException as e:
if e.status != 406 and e.status != 431:
raise
# CODE_USER_NOTFOUND or CODE_CONTAINER_NOTFOUND
metadata = dict()
metadata["dtime"] = time.time()
self.account.container_update(account, container, metadata)
@handle_account_not_found
def account_refresh(self, account, **kwargs):
self.account.account_refresh(account)
containers = self.container_list(account)
for container in containers:
try:
self.container_refresh(account, container[0])
except exc.NoSuchContainer:
# container remove in the meantime
pass
while containers:
marker = containers[-1][0]
containers = self.container_list(account, marker=marker)
if containers:
for container in containers:
try:
self.container_refresh(account, container[0])
except exc.NoSuchContainer:
# container remove in the meantime
pass
def all_accounts_refresh(self, **kwargs):
accounts = self.account_list()
for account in accounts:
try:
self.account_refresh(account)
except exc.NoSuchAccount: # account remove in the meantime
pass
@handle_account_not_found
def account_flush(self, account):
self.account.account_flush(account)
def _random_buffer(self, dictionary, n):
return ''.join(random.choice(dictionary) for _ in range(n))
def _generate_copy(self, chunks, random_hex=60):
# random_hex is the number of hexadecimals characters to generate for
# the copy path
copies = []
for c in chunks:
tmp = ''.join([c[:-random_hex],
self._random_buffer('0123456789ABCDEF',
random_hex)])
copies.append(tmp)
return copies
def _send_copy(self, targets, copies, fullpath):
headers = {"x-oio-chunk-meta-full-path": fullpath}
if not hasattr(self, "blob_client"):
from oio.blob.client import BlobClient
self.blob_client = BlobClient()
for t, c in zip(targets, copies):
self.blob_client.chunk_link(t, c, headers=headers).status
def _prepare_update_meta2(self, targets, copies, account, container,
content):
targets_beans = []
copies_beans = []
for t, c in zip(targets, copies):
targets_beans.append(self._meta2bean(t['url'], t, content))
copies_beans.append(self._meta2bean(c, t, content))
return targets_beans, copies_beans
def _meta2bean(self, url, meta, content):
return {"type": "chunk",
"id": url,
"hash": meta['hash'],
"size": int(meta["size"]),
"pos": meta["pos"],
"content": content}
class Content(object):
def __init__(self, conf, container_id, metadata, chunks, storage_method):
self.conf = conf
self.container_id = container_id
self.metadata = metadata
self.chunks = ChunksHelper(chunks)
self.storage_method = storage_method
self.logger = get_logger(self.conf)
self.cs_client = ConscienceClient(conf)
self.blob_client = BlobClient()
self.container_client = ContainerClient(self.conf)
self.content_id = self.metadata["id"]
self.stgpol = self.metadata["policy"]
self.path = self.metadata["name"]
self.length = int(self.metadata["length"])
self.version = self.metadata["version"]
self.checksum = self.metadata["hash"]
self.mime_type = self.metadata["mime_type"]
self.chunk_method = self.metadata["chunk_method"]
def _get_spare_chunk(self, chunks_notin, chunks_broken):
spare_data = {
"notin": ChunksHelper(chunks_notin, False).raw(),
"broken": ChunksHelper(chunks_broken, False).raw()
}
try:
spare_resp = self.container_client.content_spare(
cid=self.container_id,
content=self.content_id,
data=spare_data,
stgpol=self.stgpol)
except ClientException as e:
raise exc.SpareChunkException("No spare chunk (%s)" % e.message)
url_list = []
for c in spare_resp["chunks"]:
url_list.append(c["id"])
return url_list
def _update_spare_chunk(self, current_chunk, new_url):
old = [{
'type': 'chunk',
'id': current_chunk.url,
'hash': current_chunk.checksum,
'size': current_chunk.size,
'pos': current_chunk.pos,
'content': self.content_id
}]
new = [{
'type': 'chunk',
'id': new_url,
'hash': current_chunk.checksum,
'size': current_chunk.size,
'pos': current_chunk.pos,
'content': self.content_id
}]
update_data = {'old': old, 'new': new}
self.container_client.container_raw_update(cid=self.container_id,
data=update_data)
def _create_object(self):
self.container_client.content_create(cid=self.container_id,
path=self.path,
content_id=self.content_id,
stgpol=self.stgpol,
size=self.length,
checksum=self.checksum,
version=self.version,
chunk_method=self.chunk_method,
mime_type=self.mime_type,
data=self.chunks.raw())
def rebuild_chunk(self, chunk_id, allow_same_rawx=False):
raise NotImplementedError()
def create(self, stream):
raise NotImplementedError()
def fetch(self):
raise NotImplementedError()
def delete(self):
self.container_client.content_delete(cid=self.container_id,
path=self.path)
def move_chunk(self, chunk_id):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None:
raise OrphanChunk("Chunk not found in content")
other_chunks = self.chunks.filter(
metapos=current_chunk.metapos).exclude(id=chunk_id).all()
spare_urls = self._get_spare_chunk(other_chunks, [current_chunk])
self.logger.debug("copy chunk from %s to %s", current_chunk.url,
spare_urls[0])
self.blob_client.chunk_copy(current_chunk.url, spare_urls[0])
self._update_spare_chunk(current_chunk, spare_urls[0])
try:
self.blob_client.chunk_delete(current_chunk.url)
except:
self.logger.warn("Failed to delete chunk %s" % current_chunk.url)
current_chunk.url = spare_urls[0]
return current_chunk.raw()
class TestBlobAuditorFunctional(BaseTestCase):
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.test_dir = self.conf['sds_path']
rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx')
self.rawx = 'http://' + rawx_addr
self.h = hashlib.new('md5')
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = random_str(8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = random_id(64)
self.data = random_str(1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(random_str(6), len(self.data),
self.url_rand, 1)
self.content.id_container = cid_from_name(self.account,
self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0,
self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {
"hash": self.chunk.md5,
"pos": "0",
"size": self.chunk.size,
"url": self.chunk_url
}
chunk_meta = {
'content_path': self.content.path,
'container_id': self.content.id_container,
'chunk_method': 'plain/nb_copy=3',
'policy': 'TESTPOLICY',
'id': '0000',
'version': 1,
'chunk_id': self.chunk.id_chunk,
'chunk_pos': self.chunk.pos,
'chunk_hash': self.chunk.md5,
'full_path':
['%s/%s/%s' % (self.account, self.ref, self.content.path)],
'oio_version': OIO_VERSION
}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = self.test_dir + '/data/' + self.namespace + \
'-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
self.bad_container_id = '0' * 64
def tearDown(self):
super(TestBlobAuditorFunctional, self).tearDown()
try:
self.container_c.content_delete(self.account, self.ref,
self.content.path)
except Exception:
pass
try:
self.container_c.container_destroy(self.account, self.ref)
except Exception:
pass
try:
os.remove(self.chunk_path)
except Exception:
pass
def init_content(self):
self.container_c.content_create(self.account,
self.ref,
self.content.path,
self.chunk.size,
self.hash_rand,
data=[self.chunk_proxy])
def test_chunk_audit(self):
self.init_content()
self.auditor.chunk_audit(self.chunk_path)
def test_content_deleted(self):
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_container_deleted(self):
self.container_c.container_delete(self.account, self.ref)
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_corrupted(self):
self.init_content()
with open(self.chunk_path, "w") as f:
f.write(random_str(1280))
self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_size(self):
self.init_content()
with open(self.chunk_path, "w") as f:
f.write(random_str(320))
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_size(self):
self.init_content()
xattr.setxattr(self.chunk_path,
'user.' + chunk_xattr_keys['chunk_size'], '-1')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_hash(self):
self.init_content()
xattr.setxattr(self.chunk_path,
'user.' + chunk_xattr_keys['chunk_hash'], 'WRONG_HASH')
self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_content_path(self):
self.init_content()
xattr.setxattr(self.chunk_path,
'user.' + chunk_xattr_keys['content_path'],
'WRONG_PATH')
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_id(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.' + chunk_xattr_keys['chunk_id'],
'WRONG_ID')
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_content_container(self):
self.init_content()
xattr.setxattr(self.chunk_path,
'user.' + chunk_xattr_keys['container_id'],
self.bad_container_id)
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_position(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.chunk.position', '42')
xattr.setxattr(self.chunk_path,
'user.' + chunk_xattr_keys['chunk_pos'], '42')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_hash(self):
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.chunk.md5 = self.hash_rand
self.chunk_proxy['hash'] = self.chunk.md5
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_length(self):
self.chunk.size = 320
self.chunk_proxy['size'] = self.chunk.size
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_chunk_size(self):
self.chunk.size = 320
self.chunk_proxy['size'] = self.chunk.size
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_url(self):
self.chunk_proxy['url'] = '%s/WRONG_ID' % self.rawx
self.init_content()
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
class TestBlobAuditorFunctional(BaseTestCase):
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.test_dir = self.conf['sds_path']
self.chars = string.ascii_lowercase + string.ascii_uppercase +\
string.digits
self.chars_id = string.digits + 'ABCDEF'
self.rawx = 'http://' + self.conf["rawx"][0]['addr']
self.h = hashlib.new('md5')
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = rand_generator(self.chars, 8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = rand_generator(self.chars_id, 64)
self.data = rand_generator(self.chars, 1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(rand_generator(self.chars, 6),
len(self.data), self.url_rand, 1)
self.content.id_container = cid_from_name(self.account,
self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0,
self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {
"hash": self.chunk.md5,
"pos": "0",
"size": self.chunk.size,
"url": self.chunk_url
}
chunk_meta = {
'content_size': self.content.size,
'content_chunksnb': self.content.nb_chunks,
'content_path': self.content.path,
'content_cid': self.content.id_container,
'content_mimetype': 'application/octet-stream',
'content_chunkmethod': 'bytes',
'content_policy': 'TESTPOLICY',
'content_id': '0000',
'content_version': 1,
'chunk_id': self.chunk.id_chunk,
'chunk_pos': self.chunk.pos
}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = self.test_dir + '/data/' + self.namespace + \
'-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
self.bad_container_id = '0' * 64
def tearDown(self):
super(TestBlobAuditorFunctional, self).tearDown()
try:
self.container_c.content_delete(self.account, self.ref,
self.content.path)
except Exception:
pass
try:
self.container_c.container_destroy(self.account, self.ref)
except Exception:
pass
try:
os.remove(self.chunk_path)
except Exception:
pass
def init_content(self):
self.container_c.content_create(self.account,
self.ref,
self.content.path,
self.chunk.size,
self.hash_rand,
data=[self.chunk_proxy])
def test_chunk_audit(self):
self.init_content()
self.auditor.chunk_audit(self.chunk_path)
def test_content_deleted(self):
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_container_deleted(self):
self.container_c.container_destroy(self.account, self.ref)
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_corrupted(self):
self.init_content()
with open(self.chunk_path, "w") as f:
f.write(rand_generator(self.chars, 1280))
self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_size(self):
self.init_content()
with open(self.chunk_path, "w") as f:
f.write(rand_generator(self.chars, 320))
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_content_nbchunk(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.content.nbchunk', '42')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_size(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.chunk.size', '-1')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_hash(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.chunk.hash', 'WRONG_HASH')
self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_content_size(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.content.size', '-1')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_content_path(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.content.path', 'WRONG_PATH')
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_id(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.chunk.id', 'WRONG_ID')
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_content_container(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.content.container',
self.bad_container_id)
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_position(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.chunk.position', '42')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_hash(self):
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.chunk.md5 = self.hash_rand
self.chunk_proxy['hash'] = self.chunk.md5
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_length(self):
self.chunk.size = 320
self.chunk_proxy['size'] = self.chunk.size
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_chunk_size(self):
self.chunk.size = 320
self.chunk_proxy['size'] = self.chunk.size
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_url(self):
self.chunk_proxy['url'] = '%s/WRONG_ID' % self.rawx
self.init_content()
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_content_bad_path(self):
self.content.path = 'BAD_PATH'
self.init_content()
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
class TestBlobAuditorFunctional(BaseTestCase):
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.test_dir = self.conf['sds_path']
self.chars = string.ascii_lowercase + string.ascii_uppercase +\
string.digits
self.chars_id = string.digits + 'ABCDEF'
self.rawx = 'http://' + self.conf["rawx"][0]['addr']
self.h = hashlib.new('md5')
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = rand_generator(self.chars, 8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = rand_generator(self.chars_id, 64)
self.data = rand_generator(self.chars, 1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(
rand_generator(self.chars, 6), len(self.data), self.url_rand, 1)
self.content.id_container = cid_from_name(
self.account, self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0,
self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0",
"size": self.chunk.size,
"url": self.chunk_url}
chunk_meta = {'content_size': self.content.size,
'content_chunksnb': self.content.nb_chunks,
'content_path': self.content.path,
'content_cid': self.content.id_container,
'content_id': '0000',
'content_version': 1,
'chunk_id': self.chunk.id_chunk,
'chunk_pos': self.chunk.pos}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = self.test_dir + '/data/NS-rawx-1/' +\
self.chunk.id_chunk[0:2] + "/" + self.chunk.id_chunk
self.bad_container_id = '0'*64
def tearDown(self):
super(TestBlobAuditorFunctional, self).tearDown()
try:
self.container_c.content_delete(
self.account, self.ref, self.content.path)
except Exception:
pass
try:
self.container_c.container_destroy(self.account, self.ref)
except Exception:
pass
try:
os.remove(self.chunk_path)
except Exception:
pass
def init_content(self):
self.container_c.content_create(
self.account, self.ref, self.content.path, self.chunk.size,
self.hash_rand, data=[self.chunk_proxy])
def test_chunk_audit(self):
self.init_content()
self.auditor.chunk_audit(self.chunk_path)
def test_content_deleted(self):
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_container_deleted(self):
self.container_c.container_destroy(self.account, self.ref)
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_corrupted(self):
self.init_content()
with open(self.chunk_path, "w") as f:
f.write(rand_generator(self.chars, 1280))
self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_size(self):
self.init_content()
with open(self.chunk_path, "w") as f:
f.write(rand_generator(self.chars, 320))
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_content_nbchunk(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.content.nbchunk', '42')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_size(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.chunk.size', '-1')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_hash(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.chunk.hash', 'WRONG_HASH')
self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_content_size(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.content.size', '-1')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_content_path(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.content.path', 'WRONG_PATH')
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_id(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.chunk.id', 'WRONG_ID')
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_content_container(self):
self.init_content()
xattr.setxattr(
self.chunk_path, 'user.grid.content.container',
self.bad_container_id)
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_position(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.chunk.position', '42')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_hash(self):
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.chunk.md5 = self.hash_rand
self.chunk_proxy['hash'] = self.chunk.md5
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_length(self):
self.chunk.size = 320
self.chunk_proxy['size'] = self.chunk.size
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_chunk_size(self):
self.chunk.size = 320
self.chunk_proxy['size'] = self.chunk.size
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_url(self):
self.chunk_proxy['url'] = '%s/WRONG_ID' % self.rawx
self.init_content()
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_content_bad_path(self):
self.content.path = 'BAD_PATH'
self.init_content()
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
class TestBlobAuditorFunctional(BaseTestCase):
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.ref = random_str(8)
_, rawx_loc, rawx_addr, rawx_uuid = \
self.get_service_url('rawx')
self.rawx_id = 'http://' + (rawx_uuid if rawx_uuid else rawx_addr)
self.auditor = BlobAuditorWorker(self.conf, get_logger(None), None)
self.container_client = ContainerClient(self.conf)
self.blob_client = BlobClient(conf=self.conf)
self.container_client.container_create(self.account, self.ref)
self.content = TestContent(self.account, self.ref)
self.chunk = TestChunk(self.rawx_id, rawx_loc, self.content.size,
self.content.hash)
chunk_meta = {
'container_id': self.content.cid,
'content_path': self.content.path,
'version': self.content.version,
'id': self.content.id,
'full_path': self.content.fullpath,
'chunk_method': 'plain/nb_copy=3',
'policy': 'TESTPOLICY',
'chunk_id': self.chunk.id,
'chunk_pos': self.chunk.pos,
'chunk_hash': self.chunk.metachunk_hash,
'chunk_size': self.chunk.metachunk_size,
'metachunk_hash': self.chunk.metachunk_hash,
'metachunk_size': self.chunk.metachunk_size,
'oio_version': OIO_VERSION
}
self.blob_client.chunk_put(self.chunk.url, chunk_meta,
self.content.data)
def tearDown(self):
super(TestBlobAuditorFunctional, self).tearDown()
try:
self.container_client.content_delete(self.account, self.ref,
self.content.path)
except Exception:
pass
try:
self.container_client.container_delete(self.account, self.ref)
except Exception:
pass
try:
os.remove(self.chunk.path)
except Exception:
pass
def init_content(self):
chunk_proxy = {
"url": self.chunk.url,
"pos": str(self.chunk.pos),
"hash": self.chunk.metachunk_hash,
"size": self.chunk.metachunk_size
}
self.container_client.content_create(self.account,
self.ref,
self.content.path,
version=int(time.time() *
1000000),
content_id=self.content.id,
size=self.content.size,
checksum=self.content.hash,
data={'chunks': [chunk_proxy]},
stgpol="SINGLE")
def test_chunk_audit(self):
self.init_content()
self.auditor.chunk_audit(self.chunk.path, self.chunk.id)
def test_content_deleted(self):
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk.path, self.chunk.id)
def test_container_deleted(self):
self.container_client.container_delete(self.account, self.ref)
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk.path, self.chunk.id)
def test_chunk_corrupted(self):
self.init_content()
with open(self.chunk.path, "wb") as outf:
outf.write(os.urandom(1280))
self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit,
self.chunk.path, self.chunk.id)
def test_chunk_bad_chunk_size(self):
self.init_content()
with open(self.chunk.path, "wb") as outf:
outf.write(os.urandom(320))
exc_class = (exc.FaultyChunk, exc.CorruptedChunk)
self.assertRaises(exc_class, self.auditor.chunk_audit, self.chunk.path,
self.chunk.id)
def test_xattr_bad_xattr_metachunk_size(self):
self.init_content()
xattr.setxattr(self.chunk.path,
'user.' + CHUNK_XATTR_KEYS['metachunk_size'], b'320')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk.path, self.chunk.id)
def test_xattr_bad_xattr_metachunk_hash(self):
self.init_content()
xattr.setxattr(self.chunk.path,
'user.' + CHUNK_XATTR_KEYS['metachunk_hash'],
b'0123456789ABCDEF0123456789ABCDEF')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk.path, self.chunk.id)
def test_xattr_bad_xattr_chunk_id(self):
self.init_content()
xattr.removexattr(
self.chunk.path,
'user.' + CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + str(self.chunk.id))
xattr.setxattr(
self.chunk.path,
'user.' + CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + 'WRONG_ID',
self.content.fullpath.encode('utf-8'))
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk.path, self.chunk.id)
def test_xattr_bad_xattr_content_container(self):
self.init_content()
xattr.setxattr(
self.chunk.path,
'user.' + CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + str(self.chunk.id),
encode_fullpath(self.account, 'WRONG_REF', self.content.path,
self.content.version,
self.content.id).encode('utf-8'))
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk.path, self.chunk.id)
def test_xattr_bad_xattr_content_id(self):
self.init_content()
xattr.setxattr(
self.chunk.path,
'user.' + CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + str(self.chunk.id),
encode_fullpath(self.account, self.ref, self.content.path,
self.content.version,
'0123456789ABCDEF').encode('utf-8'))
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk.path, self.chunk.id)
def test_xattr_bad_xattr_chunk_position(self):
self.init_content()
xattr.setxattr(self.chunk.path,
'user.' + CHUNK_XATTR_KEYS['chunk_pos'], b'42')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk.path, self.chunk.id)
def test_chunk_bad_meta2_metachunk_size(self):
self.content.size = 320
self.chunk.metachunk_size = 320
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk.path, self.chunk.id)
def test_chunk_bad_meta2_metachunk_hash(self):
self.chunk.metachunk_hash = '0123456789ABCDEF0123456789ABCDEF'
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk.path, self.chunk.id)
def test_chunk_bad_meta2_chunk_url(self):
self.chunk.url = '%s/0123456789ABCDEF' % self.rawx_id
self.init_content()
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk.path, self.chunk.id)
class Content(object):
def __init__(self, conf, container_id, metadata, chunks, stgpol_args):
self.conf = conf
self.container_id = container_id
self.metadata = metadata
self.chunks = ChunksHelper(chunks)
self.stgpol_args = stgpol_args
self.logger = get_logger(self.conf)
self.cs_client = ConscienceClient(conf)
self.container_client = ContainerClient(self.conf)
self.blob_client = BlobClient()
self.session = requests.Session()
self.content_id = self.metadata["id"]
self.stgpol_name = self.metadata["policy"]
self.path = self.metadata["name"]
self.length = int(self.metadata["length"])
self.version = self.metadata["version"]
self.hash = self.metadata["hash"]
self.mime_type = self.metadata["mime-type"]
self.chunk_method = self.metadata["chunk-method"]
def _meta2_get_spare_chunk(self, chunks_notin, chunks_broken):
spare_data = {
"notin": ChunksHelper(chunks_notin, False).raw(),
"broken": ChunksHelper(chunks_broken, False).raw()
}
try:
spare_resp = self.container_client.content_spare(
cid=self.container_id, content=self.content_id,
data=spare_data, stgpol=self.stgpol_name)
except ClientException as e:
raise exc.SpareChunkException("No spare chunk (%s)" % e.message)
url_list = []
for c in spare_resp["chunks"]:
url_list.append(c["id"])
return url_list
def _meta2_update_spare_chunk(self, current_chunk, new_url):
old = [{'type': 'chunk',
'id': current_chunk.url,
'hash': current_chunk.hash,
'size': current_chunk.size,
'pos': current_chunk.pos,
'content': self.content_id}]
new = [{'type': 'chunk',
'id': new_url,
'hash': current_chunk.hash,
'size': current_chunk.size,
'pos': current_chunk.pos,
'content': self.content_id}]
update_data = {'old': old, 'new': new}
self.container_client.container_raw_update(
cid=self.container_id, data=update_data)
def _meta2_create_object(self):
self.container_client.content_create(cid=self.container_id,
path=self.path,
content_id=self.content_id,
stgpol=self.stgpol_name,
size=self.length,
checksum=self.hash,
version=self.version,
chunk_method=self.chunk_method,
mime_type=self.mime_type,
data=self.chunks.raw())
def rebuild_chunk(self, chunk_id):
raise NotImplementedError()
def upload(self, stream):
try:
self._upload(stream)
except:
# Keep the stack trace
exc_info = sys.exc_info()
for chunk in self.chunks:
try:
self.blob_client.chunk_delete(chunk.url)
except:
self.logger.warn("Failed to delete %s", chunk.url)
# Raise with the original stack trace
raise exc_info[0], exc_info[1], exc_info[2]
def _upload(self, stream):
raise NotImplementedError()
def download(self):
raise NotImplementedError()
def delete(self):
self.container_client.content_delete(cid=self.container_id,
path=self.path)
def move_chunk(self, chunk_id):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None:
raise OrphanChunk("Chunk not found in content")
other_chunks = self.chunks.filter(
metapos=current_chunk.metapos).exclude(id=chunk_id).all()
spare_urls = self._meta2_get_spare_chunk(other_chunks,
[current_chunk])
self.logger.debug("copy chunk from %s to %s",
current_chunk.url, spare_urls[0])
self.blob_client.chunk_copy(current_chunk.url, spare_urls[0])
self._meta2_update_spare_chunk(current_chunk, spare_urls[0])
try:
self.blob_client.chunk_delete(current_chunk.url)
except:
self.logger.warn("Failed to delete chunk %s" % current_chunk.url)
current_chunk.url = spare_urls[0]
return current_chunk.raw()
class ObjectStorageApi(object):
"""
The Object Storage API.
High level API that wraps `AccountClient`, `ContainerClient` and
`DirectoryClient` classes.
"""
def __init__(self, namespace, **kwargs):
"""
Initialize the object storage API.
:param namespace: name of the namespace to interract with
:type namespace: `str`
:keyword connection_timeout: connection timeout towards rawx services
:type connection_timeout: `float` seconds
:keyword read_timeout: timeout for rawx responses and data reads from
the caller (when uploading)
:type read_timeout: `float` seconds
:keyword write_timeout: timeout for rawx write requests
:type write_timeout: `float` seconds
"""
self.namespace = namespace
self.connection_timeout = utils.float_value(
kwargs.get("connection_timeout"), None)
self.read_timeout = utils.float_value(kwargs.get("read_timeout"), None)
self.write_timeout = utils.float_value(kwargs.get("write_timeout"),
None)
# FIXME: share session between all the clients
self.directory = DirectoryClient({"namespace": self.namespace},
**kwargs)
self.account = AccountClient({"namespace": self.namespace}, **kwargs)
self.container = ContainerClient({"namespace": self.namespace},
**kwargs)
def account_create(self, account, headers=None):
"""
Create an account.
:param account: name of the account to create
:type account: `str`
:returns: `True` if the account has been created
"""
return self.account.account_create(account, headers=headers)
@handle_account_not_found
def account_delete(self, account, headers=None):
"""
Delete an account.
:param account: name of the account to delete
:type account: `str`
"""
self.account.account_delete(account, headers=headers)
@handle_account_not_found
def account_show(self, account, headers=None):
"""
Get information about an account.
"""
return self.account.account_show(account, headers=headers)
def account_list(self, headers=None):
"""
List accounts
"""
return self.account.account_list(headers=headers)
# FIXME:
@handle_account_not_found
def account_update(self, account, metadata, to_delete=None, headers=None):
self.account.account_update(account,
metadata,
to_delete,
headers=headers)
@handle_account_not_found
def account_set_properties(self, account, properties, headers=None):
self.account_update(account, properties, headers=headers)
@handle_account_not_found
def account_del_properties(self, account, properties, headers=None):
self.account_update(account, None, properties, headers=headers)
def container_create(self,
account,
container,
properties=None,
headers=None,
**kwargs):
"""
Create a container.
:param account: account in which to create the container
:type account: `str`
:param container: name of the container
:type container: `str`
:param properties: properties to set on the container
:type properties: `dict`
:keyword headers: extra headers to send to the proxy
:type headers: `dict`
:returns: True if the container has been created,
False if it already exists
"""
return self.container.container_create(account,
container,
properties=properties,
headers=headers,
autocreate=True,
**kwargs)
@handle_container_not_found
def container_touch(self, account, container, headers=None, **kwargs):
"""
Trigger a notification about the container state.
:param account: account from which to delete the container
:type account: `str`
:param container: name of the container
:type container: `str`
:keyword headers: extra headers to send to the proxy
:type headers: `dict`
"""
if not headers:
headers = dict()
if 'X-oio-req-id' not in headers:
headers['X-oio-req-id'] = utils.request_id()
self.container.container_touch(account,
container,
headers=headers,
**kwargs)
def container_create_many(self,
account,
containers,
properties=None,
headers=None,
**kwargs):
"""
Create Many containers
:param account: account in which to create the containers
:type account: `str`
:param containers: names of the containers
:type containers: `list`
:param properties: properties to set on the containers
:type properties: `dict`
:keyword headers: extra headers to send to the proxy
:type headers: `dict`
"""
return self.container.container_create_many(account,
containers,
properties=properties,
headers=headers,
autocreate=True,
**kwargs)
@handle_container_not_found
def container_delete(self, account, container, headers=None, **kwargs):
"""
Delete a container.
:param account: account from which to delete the container
:type account: `str`
:param container: name of the container
:type container: `str`
:keyword headers: extra headers to send to the proxy
:type headers: `dict`
"""
self.container.container_delete(account,
container,
headers=headers,
**kwargs)
@handle_account_not_found
def container_list(self,
account,
limit=None,
marker=None,
end_marker=None,
prefix=None,
delimiter=None,
headers=None):
"""
Get the list of containers of an account.
:param account: account from which to get the container list
:type account: `str`
:keyword limit: maximum number of results to return
:type limit: `int`
:keyword marker: name of the container from where to start the listing
:type marker: `str`
:keyword end_marker:
:keyword prefix:
:keyword delimiter:
:keyword headers: extra headers to send to the proxy
:type headers: `dict`
"""
resp = self.account.container_list(account,
limit=limit,
marker=marker,
end_marker=end_marker,
prefix=prefix,
delimiter=delimiter,
headers=headers)
return resp["listing"]
@handle_container_not_found
def container_show(self, account, container, headers=None):
"""
Get information about a container (user properties).
:param account: account in which the container is
:type account: `str`
:param container: name of the container
:type container: `str`
:keyword headers: extra headers to send to the proxy
:type headers: `dict`
:returns: a `dict` with "properties" containing a `dict`
of user properties.
"""
return self.container.container_show(account,
container,
headers=headers)
@handle_container_not_found
def container_get_properties(self,
account,
container,
properties=None,
headers=None):
"""
Get information about a container (user and system properties).
:param account: account in which the container is
:type account: `str`
:param container: name of the container
:type container: `str`
:param properties: *ignored*
:keyword headers: extra headers to send to the proxy
:type headers: `dict`
:returns: a `dict` with "properties" and "system" entries,
containing respectively a `dict` of user properties and
a `dict` of system properties.
"""
return self.container.container_get_properties(account,
container,
properties=properties,
headers=headers)
@handle_container_not_found
def container_set_properties(self,
account,
container,
properties=None,
clear=False,
headers=None,
**kwargs):
"""
Set properties on a container.
:param account: name of the account
:type account: `str`
:param container: name of the container where to set properties
:type container: `str`
:param properties: a dictionary of properties
:type properties: `dict`
:param clear:
:type clear: `bool`
:param headers: extra headers to pass to the proxy
:type headers: `dict`
:keyword system: dictionary of system properties to set
"""
return self.container.container_set_properties(account,
container,
properties,
clear=clear,
headers=headers,
**kwargs)
@handle_container_not_found
def container_del_properties(self,
account,
container,
properties,
headers=None,
**kwargs):
return self.container.container_del_properties(account,
container,
properties,
headers=headers,
**kwargs)
def container_update(self,
account,
container,
metadata,
clear=False,
headers=None):
if not metadata:
self.container_del_properties(account,
container, [],
headers=headers)
else:
self.container_set_properties(account,
container,
metadata,
clear,
headers=headers)
@handle_container_not_found
def object_create(self,
account,
container,
file_or_path=None,
data=None,
etag=None,
obj_name=None,
mime_type=None,
metadata=None,
policy=None,
headers=None,
key_file=None,
**_kwargs):
"""
Create an object in *container* of *account* with data taken from
either *data* (`str` or `generator`) or *file_or_path* (path to a file
or file-like object).
The object will be named after *obj_name* if specified, or after
the base name of *file_or_path*.
:param account: name of the account where to create the object
:type account: `str`
:param container: name of the container where to create the object
:type container: `str`
:param file_or_path: file-like object or path to a file from which
to read object data
:type file_or_path: `str` or file-like object
:param data: object data (if `file_or_path` is not set)
:type data: `str` or `generator`
:keyword etag: entity tag of the object
:type etag: `str`
:keyword obj_name: name of the object to create. If not set, will use
the base name of `file_or_path`.
:keyword mime_type: MIME type of the object
:type mime_type: `str`
:keyword properties: a dictionary of properties
:type properties: `dict`
:keyword policy: name of the storage policy
:type policy: `str`
:param headers: extra headers to pass to the proxy
:type headers: `dict`
:keyword key_file:
"""
if (data, file_or_path) == (None, None):
raise exc.MissingData()
src = data if data is not None else file_or_path
if src is file_or_path:
if isinstance(file_or_path, basestring):
if not os.path.exists(file_or_path):
raise exc.FileNotFound("File '%s' not found." %
file_or_path)
file_name = os.path.basename(file_or_path)
else:
try:
file_name = os.path.basename(file_or_path.name)
except AttributeError:
file_name = None
obj_name = obj_name or file_name
elif isgenerator(src):
file_or_path = utils.GeneratorIO(src)
src = file_or_path
if not obj_name:
raise exc.MissingName("No name for the object has been specified")
sysmeta = {'mime_type': mime_type, 'etag': etag}
if not headers:
headers = dict()
if 'X-oio-req-id' not in headers:
headers['X-oio-req-id'] = utils.request_id()
if src is data:
return self._object_create(account,
container,
obj_name,
BytesIO(data),
sysmeta,
properties=metadata,
policy=policy,
headers=headers,
key_file=key_file)
elif hasattr(file_or_path, "read"):
return self._object_create(account,
container,
obj_name,
src,
sysmeta,
properties=metadata,
policy=policy,
headers=headers,
key_file=key_file)
else:
with open(file_or_path, "rb") as f:
return self._object_create(account,
container,
obj_name,
f,
sysmeta,
properties=metadata,
policy=policy,
headers=headers,
key_file=key_file)
def object_touch(self, account, container, obj, headers=None, **kwargs):
"""
Trigger a notification about an object
(as if it just had been created).
:param account: name of the account where to create the object
:type account: `str`
:param container: name of the container where to create the object
:type container: `str`
:param obj: name of the object to touch
:param headers: extra headers to pass to the proxy
"""
if not headers:
headers = dict()
if 'X-oio-req-id' not in headers:
headers['X-oio-req-id'] = utils.request_id()
self.container.content_touch(account,
container,
obj,
headers=headers,
**kwargs)
@handle_object_not_found
def object_delete(self, account, container, obj, headers=None, **kwargs):
if not headers:
headers = dict()
if 'X-oio-req-id' not in headers:
headers['X-oio-req-id'] = utils.request_id()
return self.container.content_delete(account,
container,
obj,
headers=headers,
**kwargs)
def object_delete_many(self,
account,
container,
objs,
headers=None,
**kwargs):
if not headers:
headers = dict()
if 'X-oio-req-id' not in headers:
headers['X-oio-req-id'] = utils.request_id()
return self.container.content_delete_many(account,
container,
objs,
headers=headers,
**kwargs)
@handle_container_not_found
def object_list(self,
account,
container,
limit=None,
marker=None,
delimiter=None,
prefix=None,
end_marker=None,
headers=None,
properties=False,
**kwargs):
"""
Lists objects inside a container.
:returns: a dict which contains
* 'objects': the list of objects
* 'prefixes': common prefixes (only if delimiter and prefix are set)
* 'properties': a dict of container properties
* 'system': system metadata
"""
_, resp_body = self.container.content_list(account,
container,
limit=limit,
marker=marker,
end_marker=end_marker,
prefix=prefix,
delimiter=delimiter,
properties=properties,
headers=headers,
**kwargs)
for obj in resp_body['objects']:
mtype = obj.get('mime-type')
if mtype:
obj['mime_type'] = mtype
del obj['mime-type']
return resp_body
# FIXME:
@handle_object_not_found
def object_locate(self, account, container, obj, headers=None):
obj_meta, body = self.container.content_locate(account, container, obj)
return obj_meta, body
def object_analyze(self, *args, **kwargs):
"""
:deprecated: use `object_locate`
"""
return self.object_locate(*args, **kwargs)
def object_fetch(self,
account,
container,
obj,
ranges=None,
headers=None,
key_file=None):
if not headers:
headers = dict()
if 'X-oio-req-id' not in headers:
headers['X-oio-req-id'] = utils.request_id()
meta, raw_chunks = self.object_locate(account,
container,
obj,
headers=headers)
chunk_method = meta['chunk_method']
storage_method = STORAGE_METHODS.load(chunk_method)
chunks = _sort_chunks(raw_chunks, storage_method.ec)
meta['container_id'] = utils.name2cid(account, container).upper()
meta['ns'] = self.namespace
if storage_method.ec:
stream = self._fetch_stream_ec(meta, chunks, ranges,
storage_method, headers)
elif storage_method.backblaze:
stream = self._fetch_stream_backblaze(meta, chunks, ranges,
storage_method, key_file)
else:
stream = self._fetch_stream(meta, chunks, ranges, storage_method,
headers)
return meta, stream
@handle_object_not_found
def object_get_properties(self, account, container, obj, headers=None):
return self.container.content_get_properties(account, container, obj)
def object_show(self, account, container, obj, headers=None):
"""
Get a description of the content along with its user properties.
:param account: name of the account in which the object is stored
:param container: name of the container in which the object is stored
:param obj: name of the object to query
:returns: a `dict` describing the object
.. python::
{'hash': '6BF60C17CC15EEA108024903B481738F',
'ctime': '1481031763',
'deleted': 'False',
'properties': {
u'projet': u'OpenIO-SDS'},
'length': '43518',
'hash_method': 'md5',
'chunk_method': 'ec/algo=liberasurecode_rs_vand,k=6,m=3',
'version': '1481031762951972',
'policy': 'EC',
'id': '20BF2194FD420500CD4729AE0B5CBC07',
'mime_type': 'application/octet-stream',
'name': 'Makefile'}
"""
return self.container.content_show(account,
container,
obj,
headers=headers)
def object_update(self,
account,
container,
obj,
metadata,
clear=False,
headers=None):
if clear:
self.object_del_properties(account,
container,
obj, [],
headers=headers)
if metadata:
self.object_set_properties(account,
container,
obj,
metadata,
headers=headers)
@handle_object_not_found
def object_set_properties(self,
account,
container,
obj,
properties,
clear=False,
headers=None,
**kwargs):
return self.container.content_set_properties(
account,
container,
obj,
properties={'properties': properties},
headers=headers,
**kwargs)
@handle_object_not_found
def object_del_properties(self,
account,
container,
obj,
properties,
headers=None,
**kwargs):
return self.container.content_del_properties(account,
container,
obj,
properties=properties,
headers=headers,
**kwargs)
# FIXME: remove and call self.container.content_prepare() directly
def _content_prepare(self,
account,
container,
obj_name,
size,
policy=None,
headers=None):
return self.container.content_prepare(account,
container,
obj_name,
size,
stgpol=policy,
autocreate=True,
headers=headers)
def _content_preparer(self,
account,
container,
obj_name,
policy=None,
headers=None):
# TODO: optimize by asking more than one metachunk at a time
obj_meta, first_body = self.container.content_prepare(account,
container,
obj_name,
size=1,
stgpol=policy,
autocreate=True,
headers=headers)
storage_method = STORAGE_METHODS.load(obj_meta['chunk_method'])
def _fix_mc_pos(chunks, mc_pos):
for chunk in chunks:
raw_pos = chunk["pos"].split(".")
if storage_method.ec:
chunk['num'] = int(raw_pos[1])
chunk["pos"] = "%d.%d" % (mc_pos, chunk['num'])
else:
chunk["pos"] = str(mc_pos)
def _metachunk_preparer():
mc_pos = 0
_fix_mc_pos(first_body, mc_pos)
yield first_body
while True:
mc_pos += 1
_, next_body = self._content_prepare(account, container,
obj_name, 1, policy,
headers)
_fix_mc_pos(next_body, mc_pos)
yield next_body
return obj_meta, _metachunk_preparer
def _object_create(self,
account,
container,
obj_name,
source,
sysmeta,
properties=None,
policy=None,
headers=None,
key_file=None):
obj_meta, chunk_prep = self._content_preparer(account,
container,
obj_name,
policy=policy,
headers=headers)
obj_meta.update(sysmeta)
obj_meta['content_path'] = obj_name
obj_meta['container_id'] = utils.name2cid(account, container).upper()
obj_meta['ns'] = self.namespace
storage_method = STORAGE_METHODS.load(obj_meta['chunk_method'])
if storage_method.ec:
handler = ECWriteHandler(
source,
obj_meta,
chunk_prep,
storage_method,
headers=headers,
write_timeout=self.write_timeout,
read_timeout=self.read_timeout,
connection_timeout=self.connection_timeout)
elif storage_method.backblaze:
backblaze_info = self._b2_credentials(storage_method, key_file)
handler = BackblazeWriteHandler(source, obj_meta, chunk_prep,
storage_method, headers,
backblaze_info)
else:
handler = ReplicatedWriteHandler(
source,
obj_meta,
chunk_prep,
storage_method,
headers=headers,
write_timeout=self.write_timeout,
read_timeout=self.read_timeout,
connection_timeout=self.connection_timeout)
final_chunks, bytes_transferred, content_checksum = handler.stream()
etag = obj_meta.get('etag')
if etag and etag.lower() != content_checksum.lower():
raise exc.EtagMismatch("given etag %s != computed %s" %
(etag, content_checksum))
obj_meta['etag'] = content_checksum
data = {'chunks': final_chunks, 'properties': properties or {}}
# FIXME: we may just pass **obj_meta
self.container.content_create(account,
container,
obj_name,
size=bytes_transferred,
checksum=content_checksum,
data=data,
content_id=obj_meta['id'],
stgpol=obj_meta['policy'],
version=obj_meta['version'],
mime_type=obj_meta['mime_type'],
chunk_method=obj_meta['chunk_method'],
headers=headers)
return final_chunks, bytes_transferred, content_checksum
def _fetch_stream(self, meta, chunks, ranges, storage_method, headers):
total_bytes = 0
headers = headers or {}
ranges = ranges or [(None, None)]
meta_range_list = get_meta_ranges(ranges, chunks)
for meta_range_dict in meta_range_list:
for pos, meta_range in meta_range_dict.iteritems():
meta_start, meta_end = meta_range
if meta_start is not None and meta_end is not None:
headers['Range'] = http_header_from_ranges([meta_range])
reader = io.ChunkReader(
iter(chunks[pos]),
io.READ_CHUNK_SIZE,
headers,
connection_timeout=self.connection_timeout,
response_timeout=self.read_timeout,
read_timeout=self.read_timeout)
try:
it = reader.get_iter()
except Exception as err:
raise exc.OioException(
"Error while downloading position %d: %s" % (pos, err))
for part in it:
for d in part['iter']:
total_bytes += len(d)
yield d
def _fetch_stream_ec(self, meta, chunks, ranges, storage_method, headers):
ranges = ranges or [(None, None)]
meta_range_list = get_meta_ranges(ranges, chunks)
for meta_range_dict in meta_range_list:
for pos, meta_range in meta_range_dict.iteritems():
meta_start, meta_end = meta_range
handler = ECChunkDownloadHandler(
storage_method,
chunks[pos],
meta_start,
meta_end,
headers,
connection_timeout=self.connection_timeout,
response_timeout=self.read_timeout,
read_timeout=self.read_timeout)
stream = handler.get_stream()
for part_info in stream:
for d in part_info['iter']:
yield d
stream.close()
def _b2_credentials(self, storage_method, key_file):
key_file = key_file or '/etc/oio/sds/b2-appkey.conf'
try:
return BackblazeUtils.get_credentials(storage_method, key_file)
except BackblazeUtilsException as err:
raise exc.ConfigurationException(str(err))
def _fetch_stream_backblaze(self, meta, chunks, ranges, storage_method,
key_file):
backblaze_info = self._b2_credentials(storage_method, key_file)
total_bytes = 0
current_offset = 0
size = None
offset = 0
for pos in range(len(chunks)):
if ranges:
offset = ranges[pos][0]
size = ranges[pos][1]
if size is None:
size = int(meta["length"])
chunk_size = int(chunks[pos][0]["size"])
if total_bytes >= size:
break
if current_offset + chunk_size > offset:
if current_offset < offset:
_offset = offset - current_offset
else:
_offset = 0
if chunk_size + total_bytes > size:
_size = size - total_bytes
else:
_size = chunk_size
handler = BackblazeChunkDownloadHandler(
meta,
chunks[pos],
_offset,
_size,
backblaze_info=backblaze_info)
stream = handler.get_stream()
if not stream:
raise exc.OioException("Error while downloading")
total_bytes += len(stream)
yield stream
current_offset += chunk_size
class TestBlobAuditorFunctional(BaseTestCase):
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.test_dir = self.conf['sds_path']
rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx')
self.rawx = 'http://' + rawx_addr
self.h = hashlib.new('md5')
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = random_str(8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = random_id(64)
self.data = random_str(1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(
random_str(6), len(self.data), self.url_rand, 1)
self.content.id_container = cid_from_name(
self.account, self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0,
self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0",
"size": self.chunk.size,
"url": self.chunk_url}
chunk_meta = {'content_path': self.content.path,
'container_id': self.content.id_container,
'content_chunkmethod': 'plain/nb_copy=3',
'content_policy': 'TESTPOLICY',
'content_id': '0000',
'content_version': 1,
'chunk_id': self.chunk.id_chunk,
'chunk_pos': self.chunk.pos}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = self.test_dir + '/data/' + self.namespace + \
'-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
self.bad_container_id = '0'*64
def tearDown(self):
super(TestBlobAuditorFunctional, self).tearDown()
try:
self.container_c.content_delete(
self.account, self.ref, self.content.path)
except Exception:
pass
try:
self.container_c.container_destroy(self.account, self.ref)
except Exception:
pass
try:
os.remove(self.chunk_path)
except Exception:
pass
def init_content(self):
self.container_c.content_create(
self.account, self.ref, self.content.path, self.chunk.size,
self.hash_rand, data=[self.chunk_proxy])
def test_chunk_audit(self):
self.init_content()
self.auditor.chunk_audit(self.chunk_path)
def test_content_deleted(self):
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_container_deleted(self):
self.container_c.container_destroy(self.account, self.ref)
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_corrupted(self):
self.init_content()
with open(self.chunk_path, "w") as f:
f.write(random_str(1280))
self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_size(self):
self.init_content()
with open(self.chunk_path, "w") as f:
f.write(random_str(320))
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_size(self):
self.init_content()
xattr.setxattr(
self.chunk_path, 'user.' + chunk_xattr_keys['chunk_size'], '-1')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_hash(self):
self.init_content()
xattr.setxattr(
self.chunk_path, 'user.' + chunk_xattr_keys['chunk_hash'],
'WRONG_HASH')
self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_content_path(self):
self.init_content()
xattr.setxattr(
self.chunk_path, 'user.' + chunk_xattr_keys['content_path'],
'WRONG_PATH')
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_id(self):
self.init_content()
xattr.setxattr(
self.chunk_path, 'user.' + chunk_xattr_keys['chunk_id'],
'WRONG_ID')
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_content_container(self):
self.init_content()
xattr.setxattr(
self.chunk_path, 'user.' + chunk_xattr_keys['container_id'],
self.bad_container_id)
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_xattr_bad_chunk_position(self):
self.init_content()
xattr.setxattr(self.chunk_path, 'user.grid.chunk.position', '42')
xattr.setxattr(
self.chunk_path, 'user.' + chunk_xattr_keys['chunk_pos'],
'42')
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_hash(self):
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.chunk.md5 = self.hash_rand
self.chunk_proxy['hash'] = self.chunk.md5
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_length(self):
self.chunk.size = 320
self.chunk_proxy['size'] = self.chunk.size
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_chunk_size(self):
self.chunk.size = 320
self.chunk_proxy['size'] = self.chunk.size
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_chunk_bad_url(self):
self.chunk_proxy['url'] = '%s/WRONG_ID' % self.rawx
self.init_content()
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
self.chunk_path)
class TestBlobAuditorFunctional(BaseTestCase):
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf["namespace"]
self.account = self.conf["account"]
self.test_dir = self.conf["sds_path"]
rawx_num, rawx_path, rawx_addr = self.get_service_url("rawx")
self.rawx = "http://" + rawx_addr
self.h = hashlib.new("md5")
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = random_str(8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = random_id(64)
self.data = random_str(1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(random_str(6), len(self.data), self.url_rand, 1)
self.content.id_container = cid_from_name(self.account, self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0, self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0", "size": self.chunk.size, "url": self.chunk_url}
chunk_meta = {
"content_path": self.content.path,
"container_id": self.content.id_container,
"chunk_method": "plain/nb_copy=3",
"policy": "TESTPOLICY",
"id": "0000",
"version": 1,
"chunk_id": self.chunk.id_chunk,
"chunk_pos": self.chunk.pos,
"chunk_hash": self.chunk.md5,
}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = (
self.test_dir
+ "/data/"
+ self.namespace
+ "-rawx-1/"
+ self.chunk.id_chunk[0:3]
+ "/"
+ self.chunk.id_chunk
)
self.bad_container_id = "0" * 64
def tearDown(self):
super(TestBlobAuditorFunctional, self).tearDown()
try:
self.container_c.content_delete(self.account, self.ref, self.content.path)
except Exception:
pass
try:
self.container_c.container_destroy(self.account, self.ref)
except Exception:
pass
try:
os.remove(self.chunk_path)
except Exception:
pass
def init_content(self):
self.container_c.content_create(
self.account, self.ref, self.content.path, self.chunk.size, self.hash_rand, data=[self.chunk_proxy]
)
def test_chunk_audit(self):
self.init_content()
self.auditor.chunk_audit(self.chunk_path)
def test_content_deleted(self):
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path)
def test_container_deleted(self):
self.container_c.container_destroy(self.account, self.ref)
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path)
def test_chunk_corrupted(self):
self.init_content()
with open(self.chunk_path, "w") as f:
f.write(random_str(1280))
self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit, self.chunk_path)
def test_chunk_bad_size(self):
self.init_content()
with open(self.chunk_path, "w") as f:
f.write(random_str(320))
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path)
def test_xattr_bad_chunk_size(self):
self.init_content()
xattr.setxattr(self.chunk_path, "user." + chunk_xattr_keys["chunk_size"], "-1")
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path)
def test_xattr_bad_chunk_hash(self):
self.init_content()
xattr.setxattr(self.chunk_path, "user." + chunk_xattr_keys["chunk_hash"], "WRONG_HASH")
self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit, self.chunk_path)
def test_xattr_bad_content_path(self):
self.init_content()
xattr.setxattr(self.chunk_path, "user." + chunk_xattr_keys["content_path"], "WRONG_PATH")
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path)
def test_xattr_bad_chunk_id(self):
self.init_content()
xattr.setxattr(self.chunk_path, "user." + chunk_xattr_keys["chunk_id"], "WRONG_ID")
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path)
def test_xattr_bad_content_container(self):
self.init_content()
xattr.setxattr(self.chunk_path, "user." + chunk_xattr_keys["container_id"], self.bad_container_id)
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path)
def test_xattr_bad_chunk_position(self):
self.init_content()
xattr.setxattr(self.chunk_path, "user.grid.chunk.position", "42")
xattr.setxattr(self.chunk_path, "user." + chunk_xattr_keys["chunk_pos"], "42")
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path)
def test_chunk_bad_hash(self):
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.chunk.md5 = self.hash_rand
self.chunk_proxy["hash"] = self.chunk.md5
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path)
def test_chunk_bad_length(self):
self.chunk.size = 320
self.chunk_proxy["size"] = self.chunk.size
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path)
def test_chunk_bad_chunk_size(self):
self.chunk.size = 320
self.chunk_proxy["size"] = self.chunk.size
self.init_content()
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path)
def test_chunk_bad_url(self):
self.chunk_proxy["url"] = "%s/WRONG_ID" % self.rawx
self.init_content()
self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path)