def test_is_object_transient_sysmeta(self):
self.assertTrue(is_object_transient_sysmeta(
'x-object-transient-sysmeta-foo'))
self.assertFalse(is_object_transient_sysmeta(
'x-object-transient-sysmeta-'))
self.assertFalse(is_object_transient_sysmeta(
'x-object-meatmeta-foo'))
def test_is_object_transient_sysmeta(self):
self.assertTrue(
rh.is_object_transient_sysmeta('x-object-transient-sysmeta-foo'))
self.assertFalse(
rh.is_object_transient_sysmeta('x-object-transient-sysmeta-'))
self.assertFalse(
rh.is_object_transient_sysmeta('x-object-meatmeta-foo'))
def load_object_metadata(self, headers):
metadata = {}
metadata.update((k.lower(), v) for k, v in headers.items()
if is_sys_or_user_meta('object', k)
or is_object_transient_sysmeta(k))
for header_key in self.allowed_headers:
if header_key in headers:
headers_lower = header_key.lower()
metadata[headers_lower] = headers[header_key]
return metadata
def _copy_headers(src, dest):
"""
Will copy desired headers from src to dest.
:params src: an instance of collections.Mapping
:params dest: an instance of collections.Mapping
"""
for k, v in src.items():
if is_sys_or_user_meta("object", k) or is_object_transient_sysmeta(k) or k.lower() == "x-delete-at":
dest[k] = v
def _copy_headers(src, dest):
"""
Will copy desired headers from src to dest.
:params src: an instance of collections.Mapping
:params dest: an instance of collections.Mapping
"""
for k, v in src.items():
if (is_sys_or_user_meta('object', k) or is_object_transient_sysmeta(k)
or k.lower() == 'x-delete-at'):
dest[k] = v
def load_object_metadata(self, headers):
metadata = {}
metadata.update(
(k.lower(), v) for k, v in headers.iteritems()
if is_sys_or_user_meta('object', k) or
is_object_transient_sysmeta(k))
for header_key in self.allowed_headers:
if header_key in headers:
headers_lower = header_key.lower()
metadata[headers_lower] = headers[header_key]
return metadata
def make_object_response(self, req, metadata, stream=None):
conditional_etag = None
if 'X-Backend-Etag-Is-At' in req.headers:
conditional_etag = metadata.get(
req.headers['X-Backend-Etag-Is-At'])
resp = Response(request=req,
conditional_response=True,
conditional_etag=conditional_etag)
if config_true_value(metadata['deleted']):
resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE
else:
resp.headers['Content-Type'] = metadata.get(
'mime_type', 'application/octet-stream')
properties = metadata.get('properties')
if properties:
for k, v in properties.items():
if is_sys_or_user_meta('object', k) or \
is_object_transient_sysmeta(k) or \
k.lower() in self.allowed_headers:
resp.headers[str(k)] = v
hash_ = metadata.get('hash')
if hash_ is not None:
hash_ = hash_.lower()
resp.headers['etag'] = hash_
resp.headers['x-object-sysmeta-version-id'] = metadata['version']
resp.last_modified = int(metadata['mtime'])
if stream:
# Whether we are bothered with ranges or not, we wrap the
# stream in order to handle exceptions.
resp.app_iter = StreamRangeIterator(req, stream)
length_ = metadata.get('length')
if length_ is not None:
length_ = int(length_)
resp.content_length = length_
resp.content_encoding = metadata.get('encoding')
resp.accept_ranges = 'bytes'
return resp
def make_object_response(self, req, metadata, stream=None):
conditional_etag = None
if 'X-Backend-Etag-Is-At' in req.headers:
conditional_etag = metadata.get(
req.headers['X-Backend-Etag-Is-At'])
resp = Response(request=req, conditional_response=True,
conditional_etag=conditional_etag)
if config_true_value(metadata['deleted']):
resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE
else:
resp.headers['Content-Type'] = metadata.get(
'mime_type', 'application/octet-stream')
properties = metadata.get('properties')
if properties:
for k, v in properties.iteritems():
if is_sys_or_user_meta('object', k) or \
is_object_transient_sysmeta(k) or \
k.lower() in self.allowed_headers:
resp.headers[str(k)] = v
hash_ = metadata.get('hash')
if hash_ is not None:
hash_ = hash_.lower()
resp.headers['etag'] = hash_
resp.headers['x-object-sysmeta-version-id'] = metadata['version']
ts = Timestamp(metadata['ctime'])
resp.last_modified = math.ceil(float(ts))
if stream:
# Whether we are bothered with ranges or not, we wrap the
# stream in order to handle exceptions.
resp.app_iter = StreamRangeIterator(req, stream)
length_ = metadata.get('length')
if length_ is not None:
length_ = int(length_)
resp.content_length = length_
resp.content_encoding = metadata.get('encoding')
resp.accept_ranges = 'bytes'
return resp
def make_object_response(self, req, metadata, stream=None, ranges=None):
conditional_etag = None
if 'X-Backend-Etag-Is-At' in req.headers:
conditional_etag = metadata.get(
req.headers['X-Backend-Etag-Is-At'])
resp = Response(request=req,
conditional_response=True,
conditional_etag=conditional_etag)
if config_true_value(metadata['deleted']):
resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE
else:
resp.headers['Content-Type'] = metadata.get(
'mime_type', 'application/octet-stream')
properties = metadata.get('properties')
if properties:
for k, v in properties.iteritems():
if is_sys_or_user_meta('object', k) or \
is_object_transient_sysmeta(k) or \
k.lower() in self.allowed_headers:
resp.headers[str(k)] = v
resp.headers['etag'] = metadata['hash'].lower()
resp.headers['x-object-sysmeta-version-id'] = metadata['version']
ts = Timestamp(metadata['ctime'])
resp.last_modified = math.ceil(float(ts))
if stream:
if ranges:
resp.app_iter = StreamRangeIterator(stream)
else:
resp.app_iter = stream
resp.content_length = int(metadata['length'])
try:
resp.content_encoding = metadata['encoding']
except KeyError:
pass
resp.accept_ranges = 'bytes'
return resp
def make_object_response(self, req, metadata, stream=None, ranges=None):
conditional_etag = None
if 'X-Backend-Etag-Is-At' in req.headers:
conditional_etag = metadata.get(
req.headers['X-Backend-Etag-Is-At'])
resp = Response(request=req, conditional_response=True,
conditional_etag=conditional_etag)
if config_true_value(metadata['deleted']):
resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE
else:
resp.headers['Content-Type'] = metadata.get(
'mime_type', 'application/octet-stream')
properties = metadata.get('properties')
if properties:
for k, v in properties.iteritems():
if is_sys_or_user_meta('object', k) or \
is_object_transient_sysmeta(k) or \
k.lower() in self.allowed_headers:
resp.headers[str(k)] = v
resp.headers['etag'] = metadata['hash'].lower()
resp.headers['x-object-sysmeta-version-id'] = metadata['version']
ts = Timestamp(metadata['ctime'])
resp.last_modified = math.ceil(float(ts))
if stream:
if ranges:
resp.app_iter = StreamRangeIterator(stream)
else:
resp.app_iter = stream
resp.content_length = int(metadata['length'])
try:
resp.content_encoding = metadata['encoding']
except KeyError:
pass
resp.accept_ranges = 'bytes'
return resp
def __call__(self, env, start_response):
method = env['REQUEST_METHOD']
if method not in self.ALLOWED_METHODS:
raise HTTPNotImplemented()
path = env['PATH_INFO']
_, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4,
rest_with_last=True)
if env.get('QUERY_STRING'):
path += '?' + env['QUERY_STRING']
if 'swift.authorize' in env:
resp = env['swift.authorize'](swob.Request(env))
if resp:
return resp(env, start_response)
req = swob.Request(env)
self.swift_sources.append(env.get('swift.source'))
self.txn_ids.append(env.get('swift.trans_id'))
try:
resp_class, raw_headers, body = self._find_response(method, path)
headers = HeaderKeyDict(raw_headers)
except KeyError:
if (env.get('QUERY_STRING')
and (method, env['PATH_INFO']) in self._responses):
resp_class, raw_headers, body = self._find_response(
method, env['PATH_INFO'])
headers = HeaderKeyDict(raw_headers)
elif method == 'HEAD' and ('GET', path) in self._responses:
resp_class, raw_headers, body = self._find_response(
'GET', path)
body = None
headers = HeaderKeyDict(raw_headers)
elif method == 'GET' and obj and path in self.uploaded:
resp_class = swob.HTTPOk
headers, body = self.uploaded[path]
else:
raise KeyError("Didn't find %r in allowed responses" % (
(method, path),))
# simulate object PUT
if method == 'PUT' and obj:
put_body = ''.join(iter(env['wsgi.input'].read, ''))
if 'swift.callback.update_footers' in env:
footers = HeaderKeyDict()
env['swift.callback.update_footers'](footers)
req.headers.update(footers)
etag = md5(put_body).hexdigest()
headers.setdefault('Etag', etag)
headers.setdefault('Content-Length', len(put_body))
# keep it for subsequent GET requests later
self.uploaded[path] = (dict(req.headers), put_body)
if "CONTENT_TYPE" in env:
self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"]
# simulate object POST
elif method == 'POST' and obj:
metadata, data = self.uploaded.get(path, ({}, None))
# select items to keep from existing...
new_metadata = dict(
(k, v) for k, v in metadata.items()
if (not is_user_meta('object', k) and not
is_object_transient_sysmeta(k)))
# apply from new
new_metadata.update(
dict((k, v) for k, v in req.headers.items()
if (is_user_meta('object', k) or
is_object_transient_sysmeta(k) or
k.lower == 'content-type')))
self.uploaded[path] = new_metadata, data
# note: tests may assume this copy of req_headers is case insensitive
# so we deliberately use a HeaderKeyDict
self._calls.append(
FakeSwiftCall(method, path, HeaderKeyDict(req.headers)))
backend_etag_header = req.headers.get('X-Backend-Etag-Is-At')
conditional_etag = None
if backend_etag_header and backend_etag_header in headers:
# Apply conditional etag overrides
conditional_etag = headers[backend_etag_header]
# range requests ought to work, hence conditional_response=True
if isinstance(body, list):
resp = resp_class(
req=req, headers=headers, app_iter=body,
conditional_response=req.method in ('GET', 'HEAD'),
conditional_etag=conditional_etag)
else:
resp = resp_class(
req=req, headers=headers, body=body,
conditional_response=req.method in ('GET', 'HEAD'),
conditional_etag=conditional_etag)
wsgi_iter = resp(env, start_response)
self.mark_opened(path)
return LeakTrackingIter(wsgi_iter, self, path)
def print_obj_metadata(metadata, drop_prefixes=False):
"""
Print out basic info and metadata from object, as returned from
:func:`swift.obj.diskfile.read_metadata`.
Metadata should include the keys: name, Content-Type, and
X-Timestamp.
Additional metadata is displayed unmodified.
:param metadata: dict of object metadata
:param drop_prefixes: if True, strip "X-Object-Meta-", "X-Object-Sysmeta-",
and "X-Object-Transient-Sysmeta-" when displaying
User Metadata, System Metadata, and Transient
System Metadata entries
:raises ValueError:
"""
user_metadata = {}
sys_metadata = {}
transient_sys_metadata = {}
other_metadata = {}
if not metadata:
raise ValueError('Metadata is None')
path = metadata.pop('name', '')
content_type = metadata.pop('Content-Type', '')
ts = Timestamp(metadata.pop('X-Timestamp', 0))
account = container = obj = obj_hash = None
if path:
try:
account, container, obj = path.split('/', 3)[1:]
except ValueError:
raise ValueError('Path is invalid for object %r' % path)
else:
obj_hash = hash_path(account, container, obj)
print('Path: %s' % path)
print(' Account: %s' % account)
print(' Container: %s' % container)
print(' Object: %s' % obj)
print(' Object hash: %s' % obj_hash)
else:
print('Path: Not found in metadata')
if content_type:
print('Content-Type: %s' % content_type)
else:
print('Content-Type: Not found in metadata')
if ts:
print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal))
else:
print('Timestamp: Not found in metadata')
for key, value in metadata.items():
if is_user_meta('Object', key):
if drop_prefixes:
key = strip_user_meta_prefix('Object', key)
user_metadata[key] = value
elif is_sys_meta('Object', key):
if drop_prefixes:
key = strip_sys_meta_prefix('Object', key)
sys_metadata[key] = value
elif is_object_transient_sysmeta(key):
if drop_prefixes:
key = strip_object_transient_sysmeta_prefix(key)
transient_sys_metadata[key] = value
else:
other_metadata[key] = value
def print_metadata(title, items):
print(title)
if items:
for key, value in sorted(items.items()):
print(' %s: %s' % (key, value))
else:
print(' No metadata found')
print_metadata('System Metadata:', sys_metadata)
print_metadata('Transient System Metadata:', transient_sys_metadata)
print_metadata('User Metadata:', user_metadata)
print_metadata('Other Metadata:', other_metadata)
def __call__(self, env, start_response):
method = env['REQUEST_METHOD']
if method not in self.ALLOWED_METHODS:
raise HTTPNotImplemented()
path = env['PATH_INFO']
_, acc, cont, obj = split_path(env['PATH_INFO'],
0,
4,
rest_with_last=True)
if env.get('QUERY_STRING'):
path += '?' + env['QUERY_STRING']
if 'swift.authorize' in env:
resp = env['swift.authorize'](swob.Request(env))
if resp:
return resp(env, start_response)
req = swob.Request(env)
self.swift_sources.append(env.get('swift.source'))
self.txn_ids.append(env.get('swift.trans_id'))
try:
resp_class, raw_headers, body = self._find_response(method, path)
headers = HeaderKeyDict(raw_headers)
except KeyError:
if (env.get('QUERY_STRING')
and (method, env['PATH_INFO']) in self._responses):
resp_class, raw_headers, body = self._find_response(
method, env['PATH_INFO'])
headers = HeaderKeyDict(raw_headers)
elif method == 'HEAD' and ('GET', path) in self._responses:
resp_class, raw_headers, body = self._find_response(
'GET', path)
body = None
headers = HeaderKeyDict(raw_headers)
elif method == 'GET' and obj and path in self.uploaded:
resp_class = swob.HTTPOk
headers, body = self.uploaded[path]
else:
raise KeyError("Didn't find %r in allowed responses" %
((method, path), ))
# simulate object PUT
if method == 'PUT' and obj:
put_body = ''.join(iter(env['wsgi.input'].read, ''))
if 'swift.callback.update_footers' in env:
footers = HeaderKeyDict()
env['swift.callback.update_footers'](footers)
req.headers.update(footers)
etag = md5(put_body).hexdigest()
headers.setdefault('Etag', etag)
headers.setdefault('Content-Length', len(put_body))
# keep it for subsequent GET requests later
self.uploaded[path] = (dict(req.headers), put_body)
if "CONTENT_TYPE" in env:
self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"]
# simulate object POST
elif method == 'POST' and obj:
metadata, data = self.uploaded.get(path, ({}, None))
# select items to keep from existing...
new_metadata = dict((k, v) for k, v in metadata.items()
if (not is_user_meta('object', k)
and not is_object_transient_sysmeta(k)))
# apply from new
new_metadata.update(
dict((k, v) for k, v in req.headers.items() if (
is_user_meta('object', k) or is_object_transient_sysmeta(k)
or k.lower == 'content-type')))
self.uploaded[path] = new_metadata, data
# note: tests may assume this copy of req_headers is case insensitive
# so we deliberately use a HeaderKeyDict
self._calls.append(
FakeSwiftCall(method, path, HeaderKeyDict(req.headers)))
# Apply conditional etag overrides
conditional_etag = resolve_etag_is_at_header(req, headers)
# range requests ought to work, hence conditional_response=True
if isinstance(body, list):
resp = resp_class(req=req,
headers=headers,
app_iter=body,
conditional_response=req.method
in ('GET', 'HEAD'),
conditional_etag=conditional_etag)
else:
resp = resp_class(req=req,
headers=headers,
body=body,
conditional_response=req.method
in ('GET', 'HEAD'),
conditional_etag=conditional_etag)
wsgi_iter = resp(env, start_response)
self.mark_opened(path)
return LeakTrackingIter(wsgi_iter, self.mark_closed, path)
def print_obj_metadata(metadata):
"""
Print out basic info and metadata from object, as returned from
:func:`swift.obj.diskfile.read_metadata`.
Metadata should include the keys: name, Content-Type, and
X-Timestamp.
Additional metadata is displayed unmodified.
:param metadata: dict of object metadata
:raises: ValueError
"""
user_metadata = {}
sys_metadata = {}
transient_sys_metadata = {}
other_metadata = {}
if not metadata:
raise ValueError('Metadata is None')
path = metadata.pop('name', '')
content_type = metadata.pop('Content-Type', '')
ts = Timestamp(metadata.pop('X-Timestamp', 0))
account = container = obj = obj_hash = None
if path:
try:
account, container, obj = path.split('/', 3)[1:]
except ValueError:
raise ValueError('Path is invalid for object %r' % path)
else:
obj_hash = hash_path(account, container, obj)
print('Path: %s' % path)
print(' Account: %s' % account)
print(' Container: %s' % container)
print(' Object: %s' % obj)
print(' Object hash: %s' % obj_hash)
else:
print('Path: Not found in metadata')
if content_type:
print('Content-Type: %s' % content_type)
else:
print('Content-Type: Not found in metadata')
if ts:
print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal))
else:
print('Timestamp: Not found in metadata')
for key, value in metadata.items():
if is_user_meta('Object', key):
user_metadata[key] = value
elif is_sys_meta('Object', key):
sys_metadata[key] = value
elif is_object_transient_sysmeta(key):
transient_sys_metadata[key] = value
else:
other_metadata[key] = value
def print_metadata(title, items):
print(title)
if items:
for meta_key in sorted(items):
print(' %s: %s' % (meta_key, items[meta_key]))
else:
print(' No metadata found')
print_metadata('System Metadata:', sys_metadata)
print_metadata('Transient System Metadata:', transient_sys_metadata)
print_metadata('User Metadata:', user_metadata)
print_metadata('Other Metadata:', other_metadata)
def __call__(self, env, start_response):
method = env["REQUEST_METHOD"]
if method not in self.ALLOWED_METHODS:
raise HTTPNotImplemented()
path = env["PATH_INFO"]
_, acc, cont, obj = split_path(env["PATH_INFO"], 0, 4, rest_with_last=True)
if env.get("QUERY_STRING"):
path += "?" + env["QUERY_STRING"]
if "swift.authorize" in env:
resp = env["swift.authorize"](swob.Request(env))
if resp:
return resp(env, start_response)
req = swob.Request(env)
self.swift_sources.append(env.get("swift.source"))
self.txn_ids.append(env.get("swift.trans_id"))
try:
resp_class, raw_headers, body = self._find_response(method, path)
headers = HeaderKeyDict(raw_headers)
except KeyError:
if env.get("QUERY_STRING") and (method, env["PATH_INFO"]) in self._responses:
resp_class, raw_headers, body = self._find_response(method, env["PATH_INFO"])
headers = HeaderKeyDict(raw_headers)
elif method == "HEAD" and ("GET", path) in self._responses:
resp_class, raw_headers, body = self._find_response("GET", path)
body = None
headers = HeaderKeyDict(raw_headers)
elif method == "GET" and obj and path in self.uploaded:
resp_class = swob.HTTPOk
headers, body = self.uploaded[path]
else:
raise KeyError("Didn't find %r in allowed responses" % ((method, path),))
# simulate object PUT
if method == "PUT" and obj:
put_body = "".join(iter(env["wsgi.input"].read, ""))
if "swift.callback.update_footers" in env:
footers = HeaderKeyDict()
env["swift.callback.update_footers"](footers)
req.headers.update(footers)
etag = md5(put_body).hexdigest()
headers.setdefault("Etag", etag)
headers.setdefault("Content-Length", len(put_body))
# keep it for subsequent GET requests later
self.uploaded[path] = (dict(req.headers), put_body)
if "CONTENT_TYPE" in env:
self.uploaded[path][0]["Content-Type"] = env["CONTENT_TYPE"]
# simulate object POST
elif method == "POST" and obj:
metadata, data = self.uploaded.get(path, ({}, None))
# select items to keep from existing...
new_metadata = dict(
(k, v)
for k, v in metadata.items()
if (not is_user_meta("object", k) and not is_object_transient_sysmeta(k))
)
# apply from new
new_metadata.update(
dict(
(k, v)
for k, v in req.headers.items()
if (is_user_meta("object", k) or is_object_transient_sysmeta(k) or k.lower == "content-type")
)
)
self.uploaded[path] = new_metadata, data
# note: tests may assume this copy of req_headers is case insensitive
# so we deliberately use a HeaderKeyDict
self._calls.append((method, path, HeaderKeyDict(req.headers)))
# range requests ought to work, hence conditional_response=True
if isinstance(body, list):
resp = resp_class(
req=req, headers=headers, app_iter=body, conditional_response=req.method in ("GET", "HEAD")
)
else:
resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ("GET", "HEAD"))
wsgi_iter = resp(env, start_response)
self.mark_opened(path)
return LeakTrackingIter(wsgi_iter, self, path)
def print_obj_metadata(metadata, drop_prefixes=False):
"""
Print out basic info and metadata from object, as returned from
:func:`swift.obj.diskfile.read_metadata`.
Metadata should include the keys: name, Content-Type, and
X-Timestamp.
Additional metadata is displayed unmodified.
:param metadata: dict of object metadata
:param drop_prefixes: if True, strip "X-Object-Meta-", "X-Object-Sysmeta-",
and "X-Object-Transient-Sysmeta-" when displaying
User Metadata, System Metadata, and Transient
System Metadata entries
:raises ValueError:
"""
user_metadata = {}
sys_metadata = {}
transient_sys_metadata = {}
other_metadata = {}
if not metadata:
raise ValueError('Metadata is None')
path = metadata.pop('name', '')
content_type = metadata.pop('Content-Type', '')
ts = Timestamp(metadata.pop('X-Timestamp', 0))
account = container = obj = obj_hash = None
if path:
try:
account, container, obj = path.split('/', 3)[1:]
except ValueError:
raise ValueError('Path is invalid for object %r' % path)
else:
obj_hash = hash_path(account, container, obj)
print('Path: %s' % path)
print(' Account: %s' % account)
print(' Container: %s' % container)
print(' Object: %s' % obj)
print(' Object hash: %s' % obj_hash)
else:
print('Path: Not found in metadata')
if content_type:
print('Content-Type: %s' % content_type)
else:
print('Content-Type: Not found in metadata')
if ts:
print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal))
else:
print('Timestamp: Not found in metadata')
for key, value in metadata.items():
if is_user_meta('Object', key):
if drop_prefixes:
key = strip_user_meta_prefix('Object', key)
user_metadata[key] = value
elif is_sys_meta('Object', key):
if drop_prefixes:
key = strip_sys_meta_prefix('Object', key)
sys_metadata[key] = value
elif is_object_transient_sysmeta(key):
if drop_prefixes:
key = strip_object_transient_sysmeta_prefix(key)
transient_sys_metadata[key] = value
else:
other_metadata[key] = value
def print_metadata(title, items):
print(title)
if items:
for key, value in sorted(items.items()):
print(' %s: %s' % (key, value))
else:
print(' No metadata found')
print_metadata('System Metadata:', sys_metadata)
print_metadata('Transient System Metadata:', transient_sys_metadata)
print_metadata('User Metadata:', user_metadata)
print_metadata('Other Metadata:', other_metadata)
for label, meta in [
('Data crypto details',
sys_metadata.get('X-Object-Sysmeta-Crypto-Body-Meta')),
('Metadata crypto details',
transient_sys_metadata.get('X-Object-Transient-Sysmeta-Crypto-Meta')),
]:
if meta is None:
continue
print('%s: %s' % (
label,
json.dumps(load_crypto_meta(meta, b64decode=False), indent=2,
sort_keys=True, separators=(',', ': '))))
