def test_is_sys_or_user_meta(self): m_types = ['sysmeta', 'meta'] for mt in m_types: for st in server_types: self.assertTrue(is_sys_or_user_meta(st, 'x-%s-%s-foo' % (st, mt))) self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%s-' % (st, mt))) self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%sfoo' % (st, mt)))
def test_is_sys_or_user_meta(self): m_types = ['sysmeta', 'meta'] for mt in m_types: for st in server_types: self.assertTrue( rh.is_sys_or_user_meta(st, 'x-%s-%s-foo' % (st, mt))) self.assertFalse( rh.is_sys_or_user_meta(st, 'x-%s-%s-' % (st, mt))) self.assertFalse( rh.is_sys_or_user_meta(st, 'x-%s-%sfoo' % (st, mt)))
def POST(self, req): """Handle HTTP POST request.""" drive, part, account, container = split_and_validate_path(req, 4) req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, self.realms_conf) if err: return HTTPBadRequest(err) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container) if broker.is_deleted(): return HTTPNotFound(request=req) metadata = {} metadata.update( (key, (value, req_timestamp.internal)) for key, value in req.headers.iteritems() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ metadata['X-Container-Sync-To'][0] != \ broker.metadata['X-Container-Sync-To'][0]: broker.set_x_container_sync_points(-1, -1) broker.update_metadata(metadata, validate_metadata=True) return HTTPNoContent(request=req)
def get_metadata_resp_headers(self, meta): headers = {} system = meta.get('system') or {} # sys.m2.ctime is microseconds ctime = float(system.get('sys.m2.ctime', 0)) / 1000000.0 headers.update({ 'X-Container-Object-Count': system.get('sys.m2.objects', 0), 'X-Container-Bytes-Used': system.get('sys.m2.usage', 0), 'X-Timestamp': Timestamp(ctime).normal, # FIXME: save modification time somewhere 'X-PUT-Timestamp': Timestamp(ctime).normal, }) for (k, v) in meta['properties'].items(): if v and (k.lower() in self.pass_through_headers or is_sys_or_user_meta('container', k)): headers[k] = v # HACK: oio-sds always sets version numbers, so let some middlewares # think that versioning is always enabled. if SYSMETA_VERSIONS_CONT not in headers and 'sys.user.name' in system: try: v_con = get_reserved_name('versions', system['sys.user.name']) headers[SYSMETA_VERSIONS_CONT] = v_con except ValueError: # sys.user.name contains reserved characters # -> this is probably a versioning container. pass return headers
def HEAD(self, request): """Handle HTTP HEAD requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = get_name_and_placement(request, 5, 5, True) try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, "timestamp"): headers["X-Backend-Timestamp"] = e.timestamp.internal return HTTPNotFound(request=request, headers=headers, conditional_response=True) response = Response(request=request, conditional_response=True) response.headers["Content-Type"] = metadata.get("Content-Type", "application/octet-stream") for key, value in metadata.iteritems(): if is_sys_or_user_meta("object", key) or key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata["ETag"] ts = Timestamp(metadata["X-Timestamp"]) response.last_modified = math.ceil(float(ts)) # Needed for container sync feature response.headers["X-Timestamp"] = ts.normal response.headers["X-Backend-Timestamp"] = ts.internal response.content_length = int(metadata["Content-Length"]) try: response.content_encoding = metadata["Content-Encoding"] except KeyError: pass return response
def HEAD(self, req): """Handle HTTP HEAD request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) out_content_type = get_listing_content_type(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() headers = { 'X-Container-Object-Count': info['object_count'], 'X-Container-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp'], } headers.update( (key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key))) headers['Content-Type'] = out_content_type return HTTPNoContent(request=req, headers=headers, charset='utf-8')
def HEAD(self, req): """Handle HTTP HEAD request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) info, is_deleted = broker.get_info_is_deleted() headers = gen_resp_headers(info, is_deleted=is_deleted) if is_deleted: return HTTPNotFound(request=req, headers=headers) headers.update( (str_to_wsgi(key), str_to_wsgi(value)) for key, (value, timestamp) in broker.metadata.items() if value != '' and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key))) headers['Content-Type'] = out_content_type resp = HTTPNoContent(request=req, headers=headers, charset='utf-8') resp.last_modified = math.ceil(float(headers['X-PUT-Timestamp'])) return resp
def _update_metadata(self, req, broker, req_timestamp): metadata = { wsgi_to_str(key): (wsgi_to_str(value), req_timestamp.internal) for key, value in req.headers.items() if is_sys_or_user_meta('account', key)} if metadata: broker.update_metadata(metadata, validate_metadata=True)
def create_listing(self, req, out_content_type, info, resp_headers, metadata, container_list, container): for key, (value, _timestamp) in metadata.items(): if value and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key)): resp_headers[str_to_wsgi(key)] = str_to_wsgi(value) listing = [ self.update_data_record(record) for record in container_list ] if out_content_type.endswith('/xml'): body = listing_formats.container_to_xml(listing, container) elif out_content_type.endswith('/json'): body = json.dumps(listing).encode('ascii') else: body = listing_formats.listing_to_text(listing) ret = Response(request=req, headers=resp_headers, body=body, content_type=out_content_type, charset='utf-8') ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp'])) if not ret.body: ret.status_int = HTTP_NO_CONTENT return ret
def HEAD(self, req): """Handle HTTP HEAD request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) out_content_type = get_listing_content_type(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) info, is_deleted = broker.get_info_is_deleted() headers = gen_resp_headers(info, is_deleted=is_deleted) if is_deleted: return HTTPNotFound(request=req, headers=headers) headers.update( (key, value) for key, (value, timestamp) in broker.metadata.items() if value != '' and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key))) headers['Content-Type'] = out_content_type resp = HTTPNoContent(request=req, headers=headers, charset='utf-8') resp.last_modified = math.ceil(float(headers['X-PUT-Timestamp'])) return resp
def HEAD(self, req): """Handle HTTP HEAD request.""" drive, part, account, container, obj = get_obj_name_and_placement(req) out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) info, is_deleted = broker.get_info_is_deleted() headers = gen_resp_headers(info, is_deleted=is_deleted) if is_deleted: return HTTPNotFound(request=req, headers=headers) headers.update( (str_to_wsgi(key), str_to_wsgi(value)) for key, (value, timestamp) in broker.metadata.items() if value != '' and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key))) headers['Content-Type'] = out_content_type resp = HTTPNoContent(request=req, headers=headers, charset='utf-8') resp.last_modified = math.ceil(float(headers['X-PUT-Timestamp'])) return resp
def make_object_response(self, req, metadata, stream=None): conditional_etag = None if 'X-Backend-Etag-Is-At' in req.headers: conditional_etag = metadata.get( req.headers['X-Backend-Etag-Is-At']) resp = Response(request=req, conditional_response=True, conditional_etag=conditional_etag) resp.headers['Content-Type'] = metadata.get( 'mime-type', 'application/octet-stream') for k, v in metadata.iteritems(): if k.startswith("user."): meta = k[5:] if is_sys_or_user_meta('object', meta) or \ meta.lower() in self.allowed_headers: resp.headers[meta] = v resp.etag = metadata['hash'].lower() ts = Timestamp(metadata['ctime']) resp.last_modified = math.ceil(float(ts)) if stream: resp.app_iter = stream resp.content_length = int(metadata['length']) try: resp.content_encoding = metadata['encoding'] except KeyError: pass return resp
def POST(self, req): """Handle HTTP POST request.""" drive, part, account, container = split_and_validate_path(req, 4) if 'x-timestamp' not in req.headers or \ not check_float(req.headers['x-timestamp']): return HTTPBadRequest(body='Missing or bad timestamp', request=req, content_type='text/plain') if 'x-container-sync-to' in req.headers: err = validate_sync_to(req.headers['x-container-sync-to'], self.allowed_sync_hosts) if err: return HTTPBadRequest(err) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container) if broker.is_deleted(): return HTTPNotFound(request=req) timestamp = normalize_timestamp(req.headers['x-timestamp']) metadata = {} metadata.update( (key, (value, timestamp)) for key, value in req.headers.iteritems() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ metadata['X-Container-Sync-To'][0] != \ broker.metadata['X-Container-Sync-To'][0]: broker.set_x_container_sync_points(-1, -1) broker.update_metadata(metadata) return HTTPNoContent(request=req)
def POST(self, req): """HTTP POST request handler.""" error_response = \ self.clean_acls(req) or check_metadata(req, 'container') if error_response: return error_response if not req.environ.get('swift_owner'): for key in self.app.swift_owner_headers: req.headers.pop(key, None) headers = self.generate_request_headers(req, transfer=True) clear_info_cache(self.app, req.environ, self.account_name, self.container_name) storage = self.app.storage metadata = {} metadata.update(("user.%s" % k, v) for k, v in req.headers.iteritems() if k.lower() in self.pass_through_headers or is_sys_or_user_meta('container', k)) try: storage.container_update(self.account_name, self.container_name, metadata, headers=headers) resp = HTTPNoContent(request=req) except exceptions.NoSuchContainer: resp = self.PUT(req) return resp
def POST(self, req): """Handle HTTP POST request.""" drive, part, account, container = split_and_validate_path(req, 4) req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, self.realms_conf) if err: return HTTPBadRequest(err) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container) if broker.is_deleted(): return HTTPNotFound(request=req) broker.update_put_timestamp(req_timestamp.internal) metadata = {} metadata.update((key, (value, req_timestamp.internal)) for key, value in req.headers.items() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ metadata['X-Container-Sync-To'][0] != \ broker.metadata['X-Container-Sync-To'][0]: broker.set_x_container_sync_points(-1, -1) broker.update_metadata(metadata, validate_metadata=True) self._update_sync_store(broker, 'POST') return HTTPNoContent(request=req)
def POST(self, req): """Handle HTTP POST request.""" drive, part, account, container = split_and_validate_path(req, 4) if 'x-timestamp' not in req.headers or \ not check_float(req.headers['x-timestamp']): return HTTPBadRequest(body='Missing or bad timestamp', request=req, content_type='text/plain') if 'x-container-sync-to' in req.headers: err = validate_sync_to(req.headers['x-container-sync-to'], self.allowed_sync_hosts) if err: return HTTPBadRequest(err) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container) if broker.is_deleted(): return HTTPNotFound(request=req) timestamp = normalize_timestamp(req.headers['x-timestamp']) metadata = {} metadata.update((key, (value, timestamp)) for key, value in req.headers.iteritems() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ metadata['X-Container-Sync-To'][0] != \ broker.metadata['X-Container-Sync-To'][0]: broker.set_x_container_sync_points(-1, -1) broker.update_metadata(metadata) return HTTPNoContent(request=req)
def HEAD(self, request): """Handle HTTP HEAD requests for the Swift on File object server""" device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) # Get DiskFile try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy=policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) # Read DiskFile metadata try: disk_file.open() metadata = disk_file.get_metadata() except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): headers['X-Backend-Timestamp'] = e.timestamp.internal return HTTPNotFound(request=request, headers=headers, conditional_respose=True) # Create and populate our response response = Response(request=request, conditional_response=True) response.headers['Content-Type'] = \ metadata.get('Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_sys_or_user_meta('object', key) or key.lower() in \ self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] ts = Timestamp(metadata['X-Timestamp']) response.last_modified = math.ceil(float(ts)) # Needed for container sync feature response.headers['X-Timestamp'] = ts.normal response.headers['X-Backend-Timestamp'] = ts.internal response.content_length = int(metadata['Content-Length']) try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass # (HPSS) Inject HPSS xattr metadata into headers want_hpss_metadata = request.headers.get('X-HPSS-Get-Metadata', False) if config_true_value(want_hpss_metadata): try: hpss_headers = disk_file.read_hpss_system_metadata() response.headers.update(hpss_headers) except SwiftOnFileSystemIOError: return HTTPServiceUnavailable(request=request) if 'X-Object-Sysmeta-Update-Container' in response.headers: self._sof_container_update(request, response) response.headers.pop('X-Object-Sysmeta-Update-Container') return response
def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy=policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) file_x_ts = Timestamp(metadata['X-Timestamp']) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) conditional_etag = None if 'X-Backend-Etag-Is-At' in request.headers: conditional_etag = metadata.get( request.headers['X-Backend-Etag-Is-At']) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True, conditional_etag=conditional_etag) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_sys_or_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(float(file_x_ts)) response.content_length = obj_size try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts.normal response.headers['X-Backend-Timestamp'] = file_x_ts.internal resp = request.get_response(response) except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): headers['X-Backend-Timestamp'] = e.timestamp.internal resp = HTTPNotFound(request=request, headers=headers, conditional_response=True) return resp
def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) print 'request',request print 'device, partition, account, container,obj, policy', device,partition,account,container,obj,policy keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile( device, partition, account, container, obj, policy=policy) print 'disk_file',disk_file except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) file_x_ts = Timestamp(metadata['X-Timestamp']) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) conditional_etag = None if 'X-Backend-Etag-Is-At' in request.headers: conditional_etag = metadata.get( request.headers['X-Backend-Etag-Is-At']) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True, conditional_etag=conditional_etag) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_sys_or_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(float(file_x_ts)) response.content_length = obj_size try: response.content_encoding = metadata[ 'Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts.normal response.headers['X-Backend-Timestamp'] = file_x_ts.internal resp = request.get_response(response) except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): headers['X-Backend-Timestamp'] = e.timestamp.internal resp = HTTPNotFound(request=request, headers=headers, conditional_response=True) print 'resp',resp return resp
def PUT(self, req): """Handle HTTP PUT request.""" drive, part, account, container = split_and_validate_path(req, 3, 4) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) if container: # put account container pending_timeout = None container_policy_index = req.headers.get(POLICY_INDEX, 0) if 'x-trans-id' in req.headers: pending_timeout = 3 broker = self._get_account_broker(drive, part, account, pending_timeout=pending_timeout) if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: broker.initialize(normalize_timestamp( req.headers.get('x-timestamp') or time.time())) except DatabaseAlreadyExists: pass if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): return HTTPNotFound(request=req) broker.put_container(container, req.headers['x-put-timestamp'], req.headers['x-delete-timestamp'], req.headers['x-object-count'], req.headers['x-bytes-used'], container_policy_index) if req.headers['x-delete-timestamp'] > \ req.headers['x-put-timestamp']: return HTTPNoContent(request=req) else: return HTTPCreated(request=req) else: # put account broker = self._get_account_broker(drive, part, account) timestamp = normalize_timestamp(req.headers['x-timestamp']) if not os.path.exists(broker.db_file): try: broker.initialize(timestamp) created = True except DatabaseAlreadyExists: created = False elif broker.is_status_deleted(): return self._deleted_response(broker, req, HTTPForbidden, body='Recently deleted') else: created = broker.is_deleted() broker.update_put_timestamp(timestamp) if broker.is_deleted(): return HTTPConflict(request=req) metadata = {} metadata.update((key, (value, timestamp)) for key, value in req.headers.iteritems() if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata) if created: return HTTPCreated(request=req) else: return HTTPAccepted(request=req)
def load_object_metadata(self, headers): metadata = {} metadata.update((k.lower(), v) for k, v in headers.iteritems() if is_sys_or_user_meta('object', k)) for header_key in self.allowed_headers: if header_key in headers: headers_lower = header_key.lower() metadata[headers_lower] = headers[header_key] return metadata
def _copy_headers(src, dest): """ Will copy desired headers from src to dest. :params src: an instance of collections.Mapping :params dest: an instance of collections.Mapping """ for k, v in src.items(): if is_sys_or_user_meta("object", k) or is_object_transient_sysmeta(k) or k.lower() == "x-delete-at": dest[k] = v
def _copy_headers_into(from_r, to_r): """ Will copy desired headers from from_r to to_r :params from_r: a swob Request or Response :params to_r: a swob Request or Response """ pass_headers = ['x-delete-at'] for k, v in from_r.headers.items(): if is_sys_or_user_meta('object', k) or k.lower() in pass_headers: to_r.headers[k] = v
def load_object_metadata(self, headers): metadata = {} metadata.update( (k.lower(), v) for k, v in headers.iteritems() if is_sys_or_user_meta('object', k) or is_object_transient_sysmeta(k)) for header_key in self.allowed_headers: if header_key in headers: headers_lower = header_key.lower() metadata[headers_lower] = headers[header_key] return metadata
def _copy_headers(src, dest): """ Will copy desired headers from src to dest. :params src: an instance of collections.Mapping :params dest: an instance of collections.Mapping """ for k, v in src.items(): if (is_sys_or_user_meta('object', k) or is_object_transient_sysmeta(k) or k.lower() == 'x-delete-at'): dest[k] = v
def PUT(self, req): """Handle HTTP PUT request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) if 'x-timestamp' not in req.headers or \ not check_float(req.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=req, content_type='text/plain') if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, self.realms_conf) if err: return HTTPBadRequest(err) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) timestamp = normalize_timestamp(req.headers['x-timestamp']) broker = self._get_container_broker(drive, part, account, container) if obj: # put container object if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: broker.initialize(timestamp) except DatabaseAlreadyExists: pass if not os.path.exists(broker.db_file): return HTTPNotFound() broker.put_object(obj, timestamp, int(req.headers['x-size']), req.headers['x-content-type'], req.headers['x-etag']) return HTTPCreated(request=req) else: # put container created = self._update_or_create(req, broker, timestamp) metadata = {} metadata.update( (key, (value, timestamp)) for key, value in req.headers.iteritems() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ metadata['X-Container-Sync-To'][0] != \ broker.metadata['X-Container-Sync-To'][0]: broker.set_x_container_sync_points(-1, -1) broker.update_metadata(metadata) resp = self.account_update(req, account, container, broker) if resp: return resp if created: return HTTPCreated(request=req) else: return HTTPAccepted(request=req)
def _update_metadata(self, req, broker, req_timestamp, method): metadata = {} metadata.update((key, (value, req_timestamp.internal)) for key, value in req.headers.items() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ metadata['X-Container-Sync-To'][0] != \ broker.metadata['X-Container-Sync-To'][0]: broker.set_x_container_sync_points(-1, -1) broker.update_metadata(metadata, validate_metadata=True) self._update_sync_store(broker, method)
def _create_response_headers(self, source_path, source_resp, sink_req): resp_headers = dict() acct, path = source_path.split('/', 3)[2:4] resp_headers['X-Copied-From-Account'] = quote(acct) resp_headers['X-Copied-From'] = quote(path) if 'last-modified' in source_resp.headers: resp_headers['X-Copied-From-Last-Modified'] = \ source_resp.headers['last-modified'] # Existing sys and user meta of source object is added to response # headers in addition to the new ones. for k, v in sink_req.headers.items(): if is_sys_or_user_meta('object', k) or k.lower() == 'x-delete-at': resp_headers[k] = v return resp_headers
def HEAD(self, request): """Handle HTTP HEAD requests for the Swift Object Server.""" device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy=policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): headers['X-Backend-Timestamp'] = e.timestamp.internal return HTTPNotFound(request=request, headers=headers, conditional_response=True) conditional_etag = None if 'X-Backend-Etag-Is-At' in request.headers: conditional_etag = metadata.get( request.headers['X-Backend-Etag-Is-At']) response = Response(request=request, conditional_response=True, conditional_etag=conditional_etag) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_sys_or_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] ts = Timestamp(metadata['X-Timestamp']) response.last_modified = math.ceil(float(ts)) # Needed for container sync feature response.headers['X-Timestamp'] = ts.normal response.headers['X-Backend-Timestamp'] = ts.internal response.content_length = int(metadata['Content-Length']) try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass return response
def create_listing(self, req, out_content_type, info, metadata, container_list, container): list_meta = get_param(req, 'list_meta', 'f').lower() in TRUE_VALUES resp_headers = { 'X-Container-Object-Count': info['object_count'], 'X-Container-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp'], } for key, (value, timestamp) in metadata.iteritems(): if value and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key)): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') if out_content_type == 'application/json': ret.body = json.dumps([self.update_data_record(record, list_meta) for record in container_list]) elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj, list_meta) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in ["name", "hash", "bytes", "content_type", "last_modified"]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') for field in sorted(record): if list_meta and field == 'metadata': meta = SubElement(obj_element, field) for k, v in record[field].iteritems(): SubElement(meta, k).text = str( v.decode('utf-8')) else: SubElement(obj_element, field).text = str( record[field]).decode('utf-8') ret.body = tostring(doc, encoding='UTF-8').replace( "<?xml version='1.0' encoding='UTF-8'?>", '<?xml version="1.0" encoding="UTF-8"?>', 1) else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) ret.body = '\n'.join(rec[0] for rec in container_list) + '\n' return ret
def _update_metadata(self, req, broker, req_timestamp, method): metadata = {} metadata.update( (wsgi_to_str(key), (wsgi_to_str(value), req_timestamp.internal)) for key, value in req.headers.items() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ metadata['X-Container-Sync-To'][0] != \ broker.metadata['X-Container-Sync-To'][0]: broker.set_x_container_sync_points(-1, -1) broker.update_metadata(metadata, validate_metadata=True) self._update_sync_store(broker, method)
def get_account_put_resp(self, req, headers): created = self.app.storage.account_create(self.account_name) metadata = {} metadata.update((key, value) for key, value in req.headers.items() if is_sys_or_user_meta('account', key)) if metadata: self.app.storage.account_update(self.account_name, metadata) if created: resp = HTTPCreated(request=req) else: resp = HTTPAccepted(request=req) return resp
def get_account_put_resp(self, req): headers = self.generate_request_headers(req, transfer=True) created = self.app.storage.account_create(self.account_name, headers=headers) metadata = {} metadata.update((k, v) for k, v in req.headers.iteritems() if is_sys_or_user_meta('account', k)) if metadata: self.app.storage.account_update(self.account_name, metadata) if created: resp = HTTPCreated(request=req) else: resp = HTTPAccepted(request=req) return resp
def POST(self, req): """Handle HTTP POST request.""" drive, part, account = split_and_validate_path(req, 3) req_timestamp = valid_timestamp(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) metadata = {} metadata.update((key, (value, req_timestamp.internal)) for key, value in req.headers.items() if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata, validate_metadata=True) return HTTPNoContent(request=req)
def POST(self, req): """Handle HTTP POST request.""" drive, part, account = split_and_validate_path(req, 3) req_timestamp = valid_timestamp(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) metadata = {} metadata.update((key, (value, req_timestamp.internal)) for key, value in req.headers.iteritems() if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata, validate_metadata=True) return HTTPNoContent(request=req)
def properties_from_headers(self, headers): metadata = { k: v for k, v in headers.items() if k.lower() in self.pass_through_headers or is_sys_or_user_meta('container', k) } system = dict() # This header enables versioning ver_loc = headers.get('X-Container-Sysmeta-Versions-Location') if ver_loc is not None: # When suspending versioning, header has empty string value ver_val = "-1" if ver_loc else "1" system['sys.m2.policy.version'] = ver_val return metadata, system
def get_metadata_resp_headers(self, meta): headers = {} system = meta.get('system') or {} # sys.m2.ctime is microseconds ctime = float(system.get('sys.m2.ctime', 0)) / 1000000.0 headers.update({ 'X-Container-Object-Count': system.get('sys.m2.objects', 0), 'X-Container-Bytes-Used': system.get('sys.m2.usage', 0), 'X-Timestamp': Timestamp(ctime).normal, # FIXME: save modification time somewhere 'X-PUT-Timestamp': Timestamp(ctime).normal, }) for (k, v) in meta['properties'].iteritems(): if v and (k.lower() in self.pass_through_headers or is_sys_or_user_meta('container', k)): headers[k] = v return headers
def POST(self, req): """Handle HTTP POST request.""" drive, part, account = split_and_validate_path(req, 3) if "x-timestamp" not in req.headers or not check_float(req.headers["x-timestamp"]): return HTTPBadRequest(body="Missing or bad timestamp", request=req, content_type="text/plain") if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) timestamp = normalize_timestamp(req.headers["x-timestamp"]) metadata = {} metadata.update( (key, (value, timestamp)) for key, value in req.headers.iteritems() if is_sys_or_user_meta("account", key) ) if metadata: broker.update_metadata(metadata) return HTTPNoContent(request=req)
def get_account_put_resp(self, req, headers): oio_headers = {'X-oio-req-id': self.trans_id} created = self.app.storage.account_create( self.account_name, headers=oio_headers) metadata = {} metadata.update((key, value) for key, value in req.headers.items() if is_sys_or_user_meta('account', key)) if metadata: self.app.storage.account_set_properties( self.account_name, metadata, headers=oio_headers) if created: resp = HTTPCreated(request=req) else: resp = HTTPAccepted(request=req) return resp
def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = get_name_and_placement(request, 5, 5, True) keep_cache = self.keep_cache_private or ( "X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers ) try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata["Content-Length"]) file_x_ts = Timestamp(metadata["X-Timestamp"]) keep_cache = self.keep_cache_private or ( "X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers ) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True ) response.headers["Content-Type"] = metadata.get("Content-Type", "application/octet-stream") for key, value in metadata.iteritems(): if is_sys_or_user_meta("object", key) or key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata["ETag"] response.last_modified = math.ceil(float(file_x_ts)) response.content_length = obj_size try: response.content_encoding = metadata["Content-Encoding"] except KeyError: pass response.headers["X-Timestamp"] = file_x_ts.normal response.headers["X-Backend-Timestamp"] = file_x_ts.internal resp = request.get_response(response) except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, "timestamp"): headers["X-Backend-Timestamp"] = e.timestamp.internal resp = HTTPNotFound(request=request, headers=headers, conditional_response=True) return resp
def get_account_post_resp(self, req, headers): metadata = {} metadata.update((key, value) for key, value in req.headers.items() if is_sys_or_user_meta('account', key)) try: self.app.storage.account_update(self.account_name, metadata) return HTTPNoContent(request=req) except (exceptions.NotFound, exceptions.NoSuchAccount): if self.app.account_autocreate: self.autocreate_account(req, self.account_name) if metadata: self.app.storage.account_update( self.account_name, metadata, headers=headers) resp = HTTPNoContent(request=req) else: resp = HTTPNotFound(request=req) self.add_acls_from_sys_metadata(resp) return resp
def POST(self, req): """Handle HTTP POST request.""" drive, part, account = split_and_validate_path(req, 3) #无用 req_timestamp = valid_timestamp(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) #获取account的数据库对象,用于存放account的数据以及元数据,获取文件对象 broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) metadata = {} #生成元数据字典,系统元数据或是用户元数据 metadata.update((key, (value, req_timestamp.internal)) for key, value in req.headers.items() if is_sys_or_user_meta('account', key)) if metadata: #如果存在需要更新的元数据,则更新元数据到数据库中,保存到文件 broker.update_metadata(metadata, validate_metadata=True) return HTTPNoContent(request=req)
def get_account_post_resp(self, req): headers = self.generate_request_headers(req, transfer=True) metadata = {} metadata.update((k, v) for k, v in req.headers.iteritems() if is_sys_or_user_meta('account', k)) try: self.app.storage.account_update(self.account_name, metadata, headers=headers) return HTTPNoContent(request=req) except exceptions.NotFound: if self.app.account_autocreate: self.autocreate_account(req, self.account_name) if metadata: self.app.storage.account_update(self.account_name, metadata, headers=headers) resp = HTTPNoContent(request=req) else: resp = HTTPNotFound(request=req) return resp
def transfer_headers(self, src_headers, dst_headers): """ Transfer legal headers from an original client request to dictionary that will be used as headers by the backend request :param src_headers: A dictionary of the original client request headers :param dst_headers: A dictionary of the backend request headers """ st = self.server_type.lower() x_remove = 'x-remove-%s-meta-' % st dst_headers.update((k.lower().replace('-remove', '', 1), '') for k in src_headers if k.lower().startswith(x_remove) or k.lower() in self._x_remove_headers()) dst_headers.update((k.lower(), v) for k, v in src_headers.iteritems() if k.lower() in self.pass_through_headers or is_sys_or_user_meta(st, k))
def create_listing(self, req, out_content_type, resp_headers, metadata, result_list, container): container_list = result_list['objects'] for p in result_list.get('prefixes', []): record = {'name': p, 'subdir': True} container_list.append(record) for (k, v) in metadata.iteritems(): if v and (k.lower() in self.pass_through_headers or is_sys_or_user_meta('container', k)): resp_headers[k] = v ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') if out_content_type == 'application/json': ret.body = json.dumps([self.update_data_record(record) for record in container_list]) elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in ["name", "hash", "bytes", "content_type", "last_modified"]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') for field in sorted(record): SubElement(obj_element, field).text = str( record[field]).decode('utf-8') ret.body = tostring(doc, encoding='UTF-8').replace( "<?xml version='1.0' encoding='UTF-8'?>", '<?xml version="1.0" encoding="UTF-8"?>', 1) else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) ret.body = '\n'.join(rec['name'] for rec in container_list) + '\n' return ret
def create_listing(self, req, out_content_type, info, resp_headers, metadata, container_list, container): for key, (value, timestamp) in metadata.items(): if value and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key)): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') if out_content_type == 'application/json': ret.body = json.dumps( [self.update_data_record(record) for record in container_list]) elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in [ "name", "hash", "bytes", "content_type", "last_modified" ]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') for field in sorted(record): SubElement(obj_element, field).text = str( record[field]).decode('utf-8') ret.body = tostring(doc, encoding='UTF-8').replace( "<?xml version='1.0' encoding='UTF-8'?>", '<?xml version="1.0" encoding="UTF-8"?>', 1) else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) ret.body = '\n'.join(rec[0] for rec in container_list) + '\n' ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp'])) return ret
def HEAD(self, req): """Handle HTTP HEAD request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) out_content_type = get_listing_content_type(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) info, is_deleted = broker.get_info_is_deleted() headers = gen_resp_headers(info, is_deleted=is_deleted) if is_deleted: return HTTPNotFound(request=req, headers=headers) headers.update( (key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key))) headers['Content-Type'] = out_content_type return HTTPNoContent(request=req, headers=headers, charset='utf-8')
def container_crawl(self, path): """ Crawls the given container path. :param path: the path to an container db """ metaDict = {} try: broker = ContainerBroker(path) if not broker.is_deleted(): #reportedTime = broker.get_info()['put_timestamp'] #if normalize_timestamp(self.crawled_time) #< reportedTime < normalize_timestamp(start_time): metaDict = broker.get_info() metaDict.update( (key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and is_sys_or_user_meta('container', key)) except (Exception, Timeout): self.logger.increment('failures') return metaDict
def POST(self, req): """Handle HTTP POST request.""" drive, part, account = split_and_validate_path(req, 3) if 'x-timestamp' not in req.headers or \ not check_float(req.headers['x-timestamp']): return HTTPBadRequest(body='Missing or bad timestamp', request=req, content_type='text/plain') if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) timestamp = normalize_timestamp(req.headers['x-timestamp']) metadata = {} metadata.update((key, (value, timestamp)) for key, value in req.headers.iteritems() if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata) return HTTPNoContent(request=req)
def create_listing(self, req, out_content_type, info, resp_headers, metadata, container_list, container): for key, (value, timestamp) in metadata.items(): if value and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key)): resp_headers[str_to_wsgi(key)] = str_to_wsgi(value) listing = [self.update_data_record(record) for record in container_list] if out_content_type.endswith('/xml'): body = listing_formats.container_to_xml(listing, container) elif out_content_type.endswith('/json'): body = json.dumps(listing).encode('ascii') else: body = listing_formats.listing_to_text(listing) ret = Response(request=req, headers=resp_headers, body=body, content_type=out_content_type, charset='utf-8') ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp'])) if not ret.body: ret.status_int = HTTP_NO_CONTENT return ret
def make_object_response(self, req, metadata, stream=None): conditional_etag = None if 'X-Backend-Etag-Is-At' in req.headers: conditional_etag = metadata.get( req.headers['X-Backend-Etag-Is-At']) resp = Response(request=req, conditional_response=True, conditional_etag=conditional_etag) if config_true_value(metadata['deleted']): resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE else: resp.headers['Content-Type'] = metadata.get( 'mime_type', 'application/octet-stream') properties = metadata.get('properties') if properties: for k, v in properties.items(): if is_sys_or_user_meta('object', k) or \ is_object_transient_sysmeta(k) or \ k.lower() in self.allowed_headers: resp.headers[str(k)] = v hash_ = metadata.get('hash') if hash_ is not None: hash_ = hash_.lower() resp.headers['etag'] = hash_ resp.headers['x-object-sysmeta-version-id'] = metadata['version'] resp.last_modified = int(metadata['mtime']) if stream: # Whether we are bothered with ranges or not, we wrap the # stream in order to handle exceptions. resp.app_iter = StreamRangeIterator(req, stream) length_ = metadata.get('length') if length_ is not None: length_ = int(length_) resp.content_length = length_ resp.content_encoding = metadata.get('encoding') resp.accept_ranges = 'bytes' return resp
def make_object_response(self, req, metadata, stream=None, ranges=None): conditional_etag = None if 'X-Backend-Etag-Is-At' in req.headers: conditional_etag = metadata.get( req.headers['X-Backend-Etag-Is-At']) resp = Response(request=req, conditional_response=True, conditional_etag=conditional_etag) if config_true_value(metadata['deleted']): resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE else: resp.headers['Content-Type'] = metadata.get( 'mime_type', 'application/octet-stream') properties = metadata.get('properties') if properties: for k, v in properties.iteritems(): if is_sys_or_user_meta('object', k) or \ is_object_transient_sysmeta(k) or \ k.lower() in self.allowed_headers: resp.headers[str(k)] = v resp.headers['etag'] = metadata['hash'].lower() resp.headers['x-object-sysmeta-version-id'] = metadata['version'] ts = Timestamp(metadata['ctime']) resp.last_modified = math.ceil(float(ts)) if stream: if ranges: resp.app_iter = StreamRangeIterator(stream) else: resp.app_iter = stream resp.content_length = int(metadata['length']) try: resp.content_encoding = metadata['encoding'] except KeyError: pass resp.accept_ranges = 'bytes' return resp
def properties_from_headers(self, headers): metadata = { k: v for k, v in headers.items() if k.lower() in self.pass_through_headers or is_sys_or_user_meta('container', k) } system = dict() # This headers enable versioning. # First the legacy one. ver_loc = headers.get('X-Container-Sysmeta-Versions-Location') if ver_loc is not None: # When suspending versioning, header has empty string value ver_val = "-1" if ver_loc else "1" system['sys.m2.policy.version'] = ver_val # Then the new one. vers_enabled = headers.get(CLIENT_VERSIONS_ENABLED) if vers_enabled is not None: ver_val = "-1" if config_true_value(vers_enabled) else "1" system['sys.m2.policy.version'] = ver_val return metadata, system