def __init__(self,
conf,
logger=None,
account_ring=None,
container_ring=None,
storage=None):
for policy, ring_arg in zip(OIO_POLICIES, RING_ARGS):
if ring_arg is not None:
policy.object_ring = FakeRing(**ring_arg)
SwiftApplication.__init__(self,
conf,
logger=logger,
account_ring=account_ring,
container_ring=container_ring)
if conf is None:
conf = dict()
sds_conf = {k[4:]: v for k, v in conf.items() if k.startswith("sds_")}
self.oio_stgpol = []
if 'auto_storage_policies' in conf:
for elem in conf['auto_storage_policies'].split(','):
if ':' in elem:
name, offset = elem.split(':')
self.oio_stgpol.append((name, int(offset)))
else:
self.oio_stgpol.append((elem, 0))
self.oio_stgpol.sort(key=lambda x: x[1])
policies = []
if 'oio_storage_policies' in conf:
for i, pol in enumerate(conf['oio_storage_policies'].split(',')):
policies.append(
storage_policy.StoragePolicy(i, pol, is_default=i == 0))
else:
policies.append(storage_policy.StoragePolicy(0, 'SINGLE', True))
self.POLICIES = storage_policy.StoragePolicyCollection(policies)
# Mandatory, raises KeyError
sds_namespace = sds_conf['namespace']
sds_conf.pop('namespace') # removed to avoid unpacking conflict
# Loaded by ObjectStorageApi if None
sds_proxy_url = sds_conf.pop('proxy_url', None)
# Fix boolean parameter
if 'autocreate' in sds_conf and not (
hasattr(ObjectStorageApi, 'EXTRA_KEYWORDS')
or 'autocreate' in ObjectStorageApi.EXTRA_KEYWORDS):
logger.warn("'autocreate' parameter is ignored by current version"
" of OpenIO SDS. Please update to oio>=4.1.23.")
else:
sds_conf['autocreate'] = config_true_value(
sds_conf.get('autocreate', 'true'))
self.storage = storage or \
ObjectStorageApi(sds_namespace, endpoint=sds_proxy_url, **sds_conf)
self.delete_slo_parts = \
config_true_value(conf.get('delete_slo_parts', False))
self.check_state = \
config_true_value(conf.get('check_state', False))
def __init__(self,
proxy_server_conf=None,
logger=None,
retries=0,
memcache=None):
self.upload_app = Application(proxy_server_conf,
memcache=memcache,
logger=logger)
self.retries = retries
def __init__(self,
conf,
memcache=None,
logger=None,
account_ring=None,
container_ring=None,
storage=None):
for policy, ring_arg in zip(POLICIES, ring_args):
if ring_arg is not None:
policy.object_ring = FakeRing(**ring_arg)
SwiftApplication.__init__(self,
conf,
memcache=memcache,
logger=logger,
account_ring=account_ring,
container_ring=container_ring)
if conf is None:
conf = dict()
sds_conf = {
k[4:]: v
for k, v in conf.iteritems() if k.startswith("sds_")
}
self.oio_stgpol = []
if 'auto_storage_policies' in conf:
for elem in conf['auto_storage_policies'].split(','):
if ':' in elem:
name, offset = elem.split(':')
self.oio_stgpol.append((name, int(offset)))
else:
self.oio_stgpol.append((elem, 0))
self.oio_stgpol.sort(key=lambda x: x[1])
policies = []
if 'oio_storage_policies' in conf:
for i, pol in enumerate(conf['oio_storage_policies'].split(',')):
policies.append(
storage_policy.StoragePolicy(i, pol, is_default=i == 0))
else:
policies.append(storage_policy.StoragePolicy(0, 'SINGLE', True))
self.POLICIES = storage_policy.StoragePolicyCollection(policies)
# Mandatory, raises KeyError
sds_namespace = sds_conf['namespace']
sds_conf.pop('namespace') # removed to avoid unpacking conflict
# Loaded by ObjectStorageApi if None
sds_proxy_url = sds_conf.pop('proxy_url', None)
self.storage = storage or \
ObjectStorageApi(sds_namespace, endpoint=sds_proxy_url, **sds_conf)
def app_factory(global_conf, **local_conf): # noqa
"""paste.deploy app factory for creating WSGI proxy apps."""
conf = global_conf.copy()
conf.update(local_conf)
return Application(conf)
def setUp(self):
self.orig_loadapp = expirer.loadapp
expirer.loadapp = lambda x: Application({},
account_ring=FakeRing(),
container_ring=FakeRing(),
object_ring=FakeRing())
def __init__(self, proxy_server_conf=None, logger=None, retries=0,
memcache=None):
self.upload_app = Application(proxy_server_conf, memcache=memcache,
logger=logger)
self.retries = retries
class InternalProxy(object):
"""
Set up a private instance of a proxy server that allows normal requests
to be made without having to actually send the request to the proxy.
This also doesn't log the requests to the normal proxy logs.
:param proxy_server_conf: proxy server configuration dictionary
:param logger: logger to log requests to
:param retries: number of times to retry each request
"""
def __init__(self, proxy_server_conf=None, logger=None, retries=0,
memcache=None):
self.upload_app = Application(proxy_server_conf, memcache=memcache,
logger=logger)
self.retries = retries
def _handle_request(self, req, source_file=None, compress=True):
req = self.upload_app.update_request(req)
req_copy = swob_request_copy(req, source_file=source_file,
compress=compress)
req_copy.headers['transfer-encoding'] = 'chunked'
resp = self.upload_app.handle_request(req_copy)
tries = 1
while (resp.status_int < 200 or resp.status_int > 299) \
and tries < self.retries:
req_copy = swob_request_copy(req, source_file=source_file,
compress=compress)
resp = self.upload_app.handle_request(req_copy)
tries += 1
return resp
def upload_file(self, source_file, account, container, object_name,
compress=True, content_type='application/x-gzip',
etag=None, headers=None):
"""
Upload a file to cloud files.
:param source_file: path to or file like object to upload
:param account: account to upload to
:param container: container to upload to
:param object_name: name of object being uploaded
:param compress: if True, compresses object as it is uploaded
:param content_type: content-type of object
:param etag: etag for object to check successful upload
:returns: True if successful, False otherwise
"""
target_name = '/v1/%s/%s/%s' % (account, container, object_name)
# create the container
if not self.create_container(account, container):
return False
send_headers = {'Transfer-Encoding': 'chunked'}
if headers:
send_headers.update(headers)
# upload the file to the account
req = swob.Request.blank(target_name,
environ={'REQUEST_METHOD': 'PUT'},
headers=send_headers)
req.environ['content_type'] = content_type
req.content_length = None # to make sure we send chunked data
if etag:
req.headers['etag'] = etag
resp = self._handle_request(req, source_file=source_file,
compress=compress)
if not (200 <= resp.status_int < 300):
return False
return True
def get_object(self, account, container, object_name):
"""
Get object.
:param account: account name object is in
:param container: container name object is in
:param object_name: name of object to get
:returns: iterator for object data
"""
req = swob.Request.blank('/v1/%s/%s/%s' %
(account, container, object_name),
environ={'REQUEST_METHOD': 'GET'})
resp = self._handle_request(req)
return resp.status_int, resp.app_iter
def create_container(self, account, container):
"""
Create container.
:param account: account name to put the container in
:param container: container name to create
:returns: True if successful, otherwise False
"""
req = swob.Request.blank('/v1/%s/%s' % (account, container),
environ={'REQUEST_METHOD': 'PUT'})
resp = self._handle_request(req)
return 200 <= resp.status_int < 300
def get_container_list(self, account, container, marker=None,
end_marker=None, limit=None, prefix=None,
delimiter=None, full_listing=True):
"""
Get a listing of objects for the container.
:param account: account name for the container
:param container: container name to get a listing for
:param marker: marker query
:param end_marker: end marker query
:param limit: limit query
:param prefix: prefix query
:param delimeter: string to delimit the queries on
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:returns: list of objects
"""
if full_listing:
rv = []
listing = self.get_container_list(account, container, marker,
end_marker, limit, prefix,
delimiter, full_listing=False)
while listing:
rv.extend(listing)
if not delimiter:
marker = listing[-1]['name']
else:
marker = listing[-1].get('name', listing[-1].get('subdir'))
listing = self.get_container_list(account, container, marker,
end_marker, limit, prefix,
delimiter,
full_listing=False)
return rv
path = '/v1/%s/%s' % (account, quote(container))
qs = 'format=json'
if marker:
qs += '&marker=%s' % quote(marker)
if end_marker:
qs += '&end_marker=%s' % quote(end_marker)
if limit:
qs += '&limit=%d' % limit
if prefix:
qs += '&prefix=%s' % quote(prefix)
if delimiter:
qs += '&delimiter=%s' % quote(delimiter)
path += '?%s' % qs
req = swob.Request.blank(path, environ={'REQUEST_METHOD': 'GET'})
resp = self._handle_request(req)
if resp.status_int < 200 or resp.status_int >= 300:
return [] # TODO: distinguish between 404 and empty container
if resp.status_int == 204:
return []
return json_loads(resp.body)
def get_container_metadata(self, account, container):
path = '/v1/%s/%s/' % (account, container)
req = swob.Request.blank(path, environ={'REQUEST_METHOD': 'HEAD'})
resp = self._handle_request(req)
out = {}
for k, v in resp.headers.iteritems():
if k.lower().startswith('x-container-meta-'):
out[k] = v
return out