def test_is_object_transient_sysmeta(self): self.assertTrue(is_object_transient_sysmeta( 'x-object-transient-sysmeta-foo')) self.assertFalse(is_object_transient_sysmeta( 'x-object-transient-sysmeta-')) self.assertFalse(is_object_transient_sysmeta( 'x-object-meatmeta-foo'))
def test_is_object_transient_sysmeta(self): self.assertTrue( rh.is_object_transient_sysmeta('x-object-transient-sysmeta-foo')) self.assertFalse( rh.is_object_transient_sysmeta('x-object-transient-sysmeta-')) self.assertFalse( rh.is_object_transient_sysmeta('x-object-meatmeta-foo'))
def load_object_metadata(self, headers): metadata = {} metadata.update((k.lower(), v) for k, v in headers.items() if is_sys_or_user_meta('object', k) or is_object_transient_sysmeta(k)) for header_key in self.allowed_headers: if header_key in headers: headers_lower = header_key.lower() metadata[headers_lower] = headers[header_key] return metadata
def _copy_headers(src, dest): """ Will copy desired headers from src to dest. :params src: an instance of collections.Mapping :params dest: an instance of collections.Mapping """ for k, v in src.items(): if is_sys_or_user_meta("object", k) or is_object_transient_sysmeta(k) or k.lower() == "x-delete-at": dest[k] = v
def _copy_headers(src, dest): """ Will copy desired headers from src to dest. :params src: an instance of collections.Mapping :params dest: an instance of collections.Mapping """ for k, v in src.items(): if (is_sys_or_user_meta('object', k) or is_object_transient_sysmeta(k) or k.lower() == 'x-delete-at'): dest[k] = v
def load_object_metadata(self, headers): metadata = {} metadata.update( (k.lower(), v) for k, v in headers.iteritems() if is_sys_or_user_meta('object', k) or is_object_transient_sysmeta(k)) for header_key in self.allowed_headers: if header_key in headers: headers_lower = header_key.lower() metadata[headers_lower] = headers[header_key] return metadata
def make_object_response(self, req, metadata, stream=None): conditional_etag = None if 'X-Backend-Etag-Is-At' in req.headers: conditional_etag = metadata.get( req.headers['X-Backend-Etag-Is-At']) resp = Response(request=req, conditional_response=True, conditional_etag=conditional_etag) if config_true_value(metadata['deleted']): resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE else: resp.headers['Content-Type'] = metadata.get( 'mime_type', 'application/octet-stream') properties = metadata.get('properties') if properties: for k, v in properties.items(): if is_sys_or_user_meta('object', k) or \ is_object_transient_sysmeta(k) or \ k.lower() in self.allowed_headers: resp.headers[str(k)] = v hash_ = metadata.get('hash') if hash_ is not None: hash_ = hash_.lower() resp.headers['etag'] = hash_ resp.headers['x-object-sysmeta-version-id'] = metadata['version'] resp.last_modified = int(metadata['mtime']) if stream: # Whether we are bothered with ranges or not, we wrap the # stream in order to handle exceptions. resp.app_iter = StreamRangeIterator(req, stream) length_ = metadata.get('length') if length_ is not None: length_ = int(length_) resp.content_length = length_ resp.content_encoding = metadata.get('encoding') resp.accept_ranges = 'bytes' return resp
def make_object_response(self, req, metadata, stream=None): conditional_etag = None if 'X-Backend-Etag-Is-At' in req.headers: conditional_etag = metadata.get( req.headers['X-Backend-Etag-Is-At']) resp = Response(request=req, conditional_response=True, conditional_etag=conditional_etag) if config_true_value(metadata['deleted']): resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE else: resp.headers['Content-Type'] = metadata.get( 'mime_type', 'application/octet-stream') properties = metadata.get('properties') if properties: for k, v in properties.iteritems(): if is_sys_or_user_meta('object', k) or \ is_object_transient_sysmeta(k) or \ k.lower() in self.allowed_headers: resp.headers[str(k)] = v hash_ = metadata.get('hash') if hash_ is not None: hash_ = hash_.lower() resp.headers['etag'] = hash_ resp.headers['x-object-sysmeta-version-id'] = metadata['version'] ts = Timestamp(metadata['ctime']) resp.last_modified = math.ceil(float(ts)) if stream: # Whether we are bothered with ranges or not, we wrap the # stream in order to handle exceptions. resp.app_iter = StreamRangeIterator(req, stream) length_ = metadata.get('length') if length_ is not None: length_ = int(length_) resp.content_length = length_ resp.content_encoding = metadata.get('encoding') resp.accept_ranges = 'bytes' return resp
def make_object_response(self, req, metadata, stream=None, ranges=None): conditional_etag = None if 'X-Backend-Etag-Is-At' in req.headers: conditional_etag = metadata.get( req.headers['X-Backend-Etag-Is-At']) resp = Response(request=req, conditional_response=True, conditional_etag=conditional_etag) if config_true_value(metadata['deleted']): resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE else: resp.headers['Content-Type'] = metadata.get( 'mime_type', 'application/octet-stream') properties = metadata.get('properties') if properties: for k, v in properties.iteritems(): if is_sys_or_user_meta('object', k) or \ is_object_transient_sysmeta(k) or \ k.lower() in self.allowed_headers: resp.headers[str(k)] = v resp.headers['etag'] = metadata['hash'].lower() resp.headers['x-object-sysmeta-version-id'] = metadata['version'] ts = Timestamp(metadata['ctime']) resp.last_modified = math.ceil(float(ts)) if stream: if ranges: resp.app_iter = StreamRangeIterator(stream) else: resp.app_iter = stream resp.content_length = int(metadata['length']) try: resp.content_encoding = metadata['encoding'] except KeyError: pass resp.accept_ranges = 'bytes' return resp
def make_object_response(self, req, metadata, stream=None, ranges=None): conditional_etag = None if 'X-Backend-Etag-Is-At' in req.headers: conditional_etag = metadata.get( req.headers['X-Backend-Etag-Is-At']) resp = Response(request=req, conditional_response=True, conditional_etag=conditional_etag) if config_true_value(metadata['deleted']): resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE else: resp.headers['Content-Type'] = metadata.get( 'mime_type', 'application/octet-stream') properties = metadata.get('properties') if properties: for k, v in properties.iteritems(): if is_sys_or_user_meta('object', k) or \ is_object_transient_sysmeta(k) or \ k.lower() in self.allowed_headers: resp.headers[str(k)] = v resp.headers['etag'] = metadata['hash'].lower() resp.headers['x-object-sysmeta-version-id'] = metadata['version'] ts = Timestamp(metadata['ctime']) resp.last_modified = math.ceil(float(ts)) if stream: if ranges: resp.app_iter = StreamRangeIterator(stream) else: resp.app_iter = stream resp.content_length = int(metadata['length']) try: resp.content_encoding = metadata['encoding'] except KeyError: pass resp.accept_ranges = 'bytes' return resp
def __call__(self, env, start_response): method = env['REQUEST_METHOD'] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) if env.get('QUERY_STRING'): path += '?' + env['QUERY_STRING'] if 'swift.authorize' in env: resp = env['swift.authorize'](swob.Request(env)) if resp: return resp(env, start_response) req = swob.Request(env) self.swift_sources.append(env.get('swift.source')) self.txn_ids.append(env.get('swift.trans_id')) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): resp_class, raw_headers, body = self._find_response( method, env['PATH_INFO']) headers = HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: resp_class, raw_headers, body = self._find_response( 'GET', path) body = None headers = HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ( (method, path),)) # simulate object PUT if method == 'PUT' and obj: put_body = ''.join(iter(env['wsgi.input'].read, '')) if 'swift.callback.update_footers' in env: footers = HeaderKeyDict() env['swift.callback.update_footers'](footers) req.headers.update(footers) etag = md5(put_body).hexdigest() headers.setdefault('Etag', etag) headers.setdefault('Content-Length', len(put_body)) # keep it for subsequent GET requests later self.uploaded[path] = (dict(req.headers), put_body) if "CONTENT_TYPE" in env: self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] # simulate object POST elif method == 'POST' and obj: metadata, data = self.uploaded.get(path, ({}, None)) # select items to keep from existing... new_metadata = dict( (k, v) for k, v in metadata.items() if (not is_user_meta('object', k) and not is_object_transient_sysmeta(k))) # apply from new new_metadata.update( dict((k, v) for k, v in req.headers.items() if (is_user_meta('object', k) or is_object_transient_sysmeta(k) or k.lower == 'content-type'))) self.uploaded[path] = new_metadata, data # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append( FakeSwiftCall(method, path, HeaderKeyDict(req.headers))) backend_etag_header = req.headers.get('X-Backend-Etag-Is-At') conditional_etag = None if backend_etag_header and backend_etag_header in headers: # Apply conditional etag overrides conditional_etag = headers[backend_etag_header] # range requests ought to work, hence conditional_response=True if isinstance(body, list): resp = resp_class( req=req, headers=headers, app_iter=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) else: resp = resp_class( req=req, headers=headers, body=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self, path)
def print_obj_metadata(metadata, drop_prefixes=False): """ Print out basic info and metadata from object, as returned from :func:`swift.obj.diskfile.read_metadata`. Metadata should include the keys: name, Content-Type, and X-Timestamp. Additional metadata is displayed unmodified. :param metadata: dict of object metadata :param drop_prefixes: if True, strip "X-Object-Meta-", "X-Object-Sysmeta-", and "X-Object-Transient-Sysmeta-" when displaying User Metadata, System Metadata, and Transient System Metadata entries :raises ValueError: """ user_metadata = {} sys_metadata = {} transient_sys_metadata = {} other_metadata = {} if not metadata: raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: account, container, obj = path.split('/', 3)[1:] except ValueError: raise ValueError('Path is invalid for object %r' % path) else: obj_hash = hash_path(account, container, obj) print('Path: %s' % path) print(' Account: %s' % account) print(' Container: %s' % container) print(' Object: %s' % obj) print(' Object hash: %s' % obj_hash) else: print('Path: Not found in metadata') if content_type: print('Content-Type: %s' % content_type) else: print('Content-Type: Not found in metadata') if ts: print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print('Timestamp: Not found in metadata') for key, value in metadata.items(): if is_user_meta('Object', key): if drop_prefixes: key = strip_user_meta_prefix('Object', key) user_metadata[key] = value elif is_sys_meta('Object', key): if drop_prefixes: key = strip_sys_meta_prefix('Object', key) sys_metadata[key] = value elif is_object_transient_sysmeta(key): if drop_prefixes: key = strip_object_transient_sysmeta_prefix(key) transient_sys_metadata[key] = value else: other_metadata[key] = value def print_metadata(title, items): print(title) if items: for key, value in sorted(items.items()): print(' %s: %s' % (key, value)) else: print(' No metadata found') print_metadata('System Metadata:', sys_metadata) print_metadata('Transient System Metadata:', transient_sys_metadata) print_metadata('User Metadata:', user_metadata) print_metadata('Other Metadata:', other_metadata)
def __call__(self, env, start_response): method = env['REQUEST_METHOD'] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) if env.get('QUERY_STRING'): path += '?' + env['QUERY_STRING'] if 'swift.authorize' in env: resp = env['swift.authorize'](swob.Request(env)) if resp: return resp(env, start_response) req = swob.Request(env) self.swift_sources.append(env.get('swift.source')) self.txn_ids.append(env.get('swift.trans_id')) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): resp_class, raw_headers, body = self._find_response( method, env['PATH_INFO']) headers = HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: resp_class, raw_headers, body = self._find_response( 'GET', path) body = None headers = HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ((method, path), )) # simulate object PUT if method == 'PUT' and obj: put_body = ''.join(iter(env['wsgi.input'].read, '')) if 'swift.callback.update_footers' in env: footers = HeaderKeyDict() env['swift.callback.update_footers'](footers) req.headers.update(footers) etag = md5(put_body).hexdigest() headers.setdefault('Etag', etag) headers.setdefault('Content-Length', len(put_body)) # keep it for subsequent GET requests later self.uploaded[path] = (dict(req.headers), put_body) if "CONTENT_TYPE" in env: self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] # simulate object POST elif method == 'POST' and obj: metadata, data = self.uploaded.get(path, ({}, None)) # select items to keep from existing... new_metadata = dict((k, v) for k, v in metadata.items() if (not is_user_meta('object', k) and not is_object_transient_sysmeta(k))) # apply from new new_metadata.update( dict((k, v) for k, v in req.headers.items() if ( is_user_meta('object', k) or is_object_transient_sysmeta(k) or k.lower == 'content-type'))) self.uploaded[path] = new_metadata, data # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append( FakeSwiftCall(method, path, HeaderKeyDict(req.headers))) # Apply conditional etag overrides conditional_etag = resolve_etag_is_at_header(req, headers) # range requests ought to work, hence conditional_response=True if isinstance(body, list): resp = resp_class(req=req, headers=headers, app_iter=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) else: resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self.mark_closed, path)
def print_obj_metadata(metadata): """ Print out basic info and metadata from object, as returned from :func:`swift.obj.diskfile.read_metadata`. Metadata should include the keys: name, Content-Type, and X-Timestamp. Additional metadata is displayed unmodified. :param metadata: dict of object metadata :raises: ValueError """ user_metadata = {} sys_metadata = {} transient_sys_metadata = {} other_metadata = {} if not metadata: raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: account, container, obj = path.split('/', 3)[1:] except ValueError: raise ValueError('Path is invalid for object %r' % path) else: obj_hash = hash_path(account, container, obj) print('Path: %s' % path) print(' Account: %s' % account) print(' Container: %s' % container) print(' Object: %s' % obj) print(' Object hash: %s' % obj_hash) else: print('Path: Not found in metadata') if content_type: print('Content-Type: %s' % content_type) else: print('Content-Type: Not found in metadata') if ts: print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print('Timestamp: Not found in metadata') for key, value in metadata.items(): if is_user_meta('Object', key): user_metadata[key] = value elif is_sys_meta('Object', key): sys_metadata[key] = value elif is_object_transient_sysmeta(key): transient_sys_metadata[key] = value else: other_metadata[key] = value def print_metadata(title, items): print(title) if items: for meta_key in sorted(items): print(' %s: %s' % (meta_key, items[meta_key])) else: print(' No metadata found') print_metadata('System Metadata:', sys_metadata) print_metadata('Transient System Metadata:', transient_sys_metadata) print_metadata('User Metadata:', user_metadata) print_metadata('Other Metadata:', other_metadata)
def __call__(self, env, start_response): method = env["REQUEST_METHOD"] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env["PATH_INFO"] _, acc, cont, obj = split_path(env["PATH_INFO"], 0, 4, rest_with_last=True) if env.get("QUERY_STRING"): path += "?" + env["QUERY_STRING"] if "swift.authorize" in env: resp = env["swift.authorize"](swob.Request(env)) if resp: return resp(env, start_response) req = swob.Request(env) self.swift_sources.append(env.get("swift.source")) self.txn_ids.append(env.get("swift.trans_id")) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if env.get("QUERY_STRING") and (method, env["PATH_INFO"]) in self._responses: resp_class, raw_headers, body = self._find_response(method, env["PATH_INFO"]) headers = HeaderKeyDict(raw_headers) elif method == "HEAD" and ("GET", path) in self._responses: resp_class, raw_headers, body = self._find_response("GET", path) body = None headers = HeaderKeyDict(raw_headers) elif method == "GET" and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ((method, path),)) # simulate object PUT if method == "PUT" and obj: put_body = "".join(iter(env["wsgi.input"].read, "")) if "swift.callback.update_footers" in env: footers = HeaderKeyDict() env["swift.callback.update_footers"](footers) req.headers.update(footers) etag = md5(put_body).hexdigest() headers.setdefault("Etag", etag) headers.setdefault("Content-Length", len(put_body)) # keep it for subsequent GET requests later self.uploaded[path] = (dict(req.headers), put_body) if "CONTENT_TYPE" in env: self.uploaded[path][0]["Content-Type"] = env["CONTENT_TYPE"] # simulate object POST elif method == "POST" and obj: metadata, data = self.uploaded.get(path, ({}, None)) # select items to keep from existing... new_metadata = dict( (k, v) for k, v in metadata.items() if (not is_user_meta("object", k) and not is_object_transient_sysmeta(k)) ) # apply from new new_metadata.update( dict( (k, v) for k, v in req.headers.items() if (is_user_meta("object", k) or is_object_transient_sysmeta(k) or k.lower == "content-type") ) ) self.uploaded[path] = new_metadata, data # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append((method, path, HeaderKeyDict(req.headers))) # range requests ought to work, hence conditional_response=True if isinstance(body, list): resp = resp_class( req=req, headers=headers, app_iter=body, conditional_response=req.method in ("GET", "HEAD") ) else: resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ("GET", "HEAD")) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self, path)
def print_obj_metadata(metadata, drop_prefixes=False): """ Print out basic info and metadata from object, as returned from :func:`swift.obj.diskfile.read_metadata`. Metadata should include the keys: name, Content-Type, and X-Timestamp. Additional metadata is displayed unmodified. :param metadata: dict of object metadata :param drop_prefixes: if True, strip "X-Object-Meta-", "X-Object-Sysmeta-", and "X-Object-Transient-Sysmeta-" when displaying User Metadata, System Metadata, and Transient System Metadata entries :raises ValueError: """ user_metadata = {} sys_metadata = {} transient_sys_metadata = {} other_metadata = {} if not metadata: raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: account, container, obj = path.split('/', 3)[1:] except ValueError: raise ValueError('Path is invalid for object %r' % path) else: obj_hash = hash_path(account, container, obj) print('Path: %s' % path) print(' Account: %s' % account) print(' Container: %s' % container) print(' Object: %s' % obj) print(' Object hash: %s' % obj_hash) else: print('Path: Not found in metadata') if content_type: print('Content-Type: %s' % content_type) else: print('Content-Type: Not found in metadata') if ts: print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print('Timestamp: Not found in metadata') for key, value in metadata.items(): if is_user_meta('Object', key): if drop_prefixes: key = strip_user_meta_prefix('Object', key) user_metadata[key] = value elif is_sys_meta('Object', key): if drop_prefixes: key = strip_sys_meta_prefix('Object', key) sys_metadata[key] = value elif is_object_transient_sysmeta(key): if drop_prefixes: key = strip_object_transient_sysmeta_prefix(key) transient_sys_metadata[key] = value else: other_metadata[key] = value def print_metadata(title, items): print(title) if items: for key, value in sorted(items.items()): print(' %s: %s' % (key, value)) else: print(' No metadata found') print_metadata('System Metadata:', sys_metadata) print_metadata('Transient System Metadata:', transient_sys_metadata) print_metadata('User Metadata:', user_metadata) print_metadata('Other Metadata:', other_metadata) for label, meta in [ ('Data crypto details', sys_metadata.get('X-Object-Sysmeta-Crypto-Body-Meta')), ('Metadata crypto details', transient_sys_metadata.get('X-Object-Transient-Sysmeta-Crypto-Meta')), ]: if meta is None: continue print('%s: %s' % ( label, json.dumps(load_crypto_meta(meta, b64decode=False), indent=2, sort_keys=True, separators=(',', ': '))))