class ObjectController(object): """Implements the WSGI application for the Swift Object Server.""" def __init__(self, conf, logger=None): """ Creates a new WSGI application for the Swift Object Server. An example configuration is given at <source-dir>/etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ self.logger = logger or get_logger(conf, log_route='object-server') self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) self.network_chunk_size = int(conf.get('network_chunk_size', 65536)) self.log_requests = config_true_value(conf.get('log_requests', 'true')) self.max_upload_time = int(conf.get('max_upload_time', 86400)) self.slow = int(conf.get('slow', 0)) self.keep_cache_private = \ config_true_value(conf.get('keep_cache_private', 'false')) replication_server = conf.get('replication_server', None) if replication_server is not None: replication_server = config_true_value(replication_server) self.replication_server = replication_server default_allowed_headers = ''' content-disposition, content-encoding, x-delete-at, x-object-manifest, x-static-large-object, ''' extra_allowed_headers = [ header.strip().lower() for header in conf.get( 'allowed_headers', default_allowed_headers).split(',') if header.strip() ] self.allowed_headers = set() for header in extra_allowed_headers: if header not in DATAFILE_SYSTEM_META: self.allowed_headers.add(header) self.expiring_objects_account = \ (conf.get('auto_create_account_prefix') or '.') + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) # Initialization was successful, so now apply the network chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because the primary motivation for this is to optimize how data # is written back to the proxy server, we could use the value from the # disk_chunk_size parameter. However, it affects all created sockets # using this class so we have chosen to tie it to the # network_chunk_size parameter value instead. socket._fileobject.default_bufsize = self.network_chunk_size # Provide further setup sepecific to an object server implemenation. self.setup(conf) def setup(self, conf): """ Implementation specific setup. This method is called at the very end by the constructor to allow a specific implementation to modify existing attributes or add its own attributes. :param conf: WSGI configuration parameter """ # Common on-disk hierarchy shared across account, container and object # servers. self._diskfile_mgr = DiskFileManager(conf, self.logger) # This is populated by global_conf_callback way below as the semaphore # is shared by all workers. if 'replication_semaphore' in conf: # The value was put in a list so it could get past paste self.replication_semaphore = conf['replication_semaphore'][0] else: self.replication_semaphore = None self.replication_failure_threshold = int( conf.get('replication_failure_threshold') or 100) self.replication_failure_ratio = float( conf.get('replication_failure_ratio') or 1.0) def get_diskfile(self, device, partition, account, container, obj, **kwargs): """ Utility method for instantiating a DiskFile object supporting a given REST API. An implementation of the object server that wants to use a different DiskFile class would simply over-ride this method to provide that behavior. """ return self._diskfile_mgr.get_diskfile( device, partition, account, container, obj, **kwargs) def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ headers_out['user-agent'] = 'obj-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error(_( 'ERROR Container update failed ' '(saving for async update later): %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), {'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice}) except (Exception, Timeout): self.logger.exception(_( 'ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s (saving for async update later)'), {'ip': ip, 'port': port, 'dev': contdevice}) data = {'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out} timestamp = headers_out['x-timestamp'] self._diskfile_mgr.pickle_async_update(objdevice, account, container, obj, data, timestamp) def container_update(self, op, account, container, obj, request, headers_out, objdevice): """ Update the container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request object driving the update :param headers_out: dictionary of headers to send in the container request(s) :param objdevice: device name that the object is in """ headers_in = request.headers conthosts = [h.strip() for h in headers_in.get('X-Container-Host', '').split(',')] contdevices = [d.strip() for d in headers_in.get('X-Container-Device', '').split(',')] contpartition = headers_in.get('X-Container-Partition', '') if len(conthosts) != len(contdevices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. self.logger.error(_('ERROR Container update failed: different ' 'numbers of hosts and devices in request: ' '"%s" vs "%s"') % (headers_in.get('X-Container-Host', ''), headers_in.get('X-Container-Device', ''))) return if contpartition: updates = zip(conthosts, contdevices) else: updates = [] headers_out['x-trans-id'] = headers_in.get('x-trans-id', '-') headers_out['referer'] = request.as_referer() for conthost, contdevice in updates: self.async_update(op, account, container, obj, conthost, contpartition, contdevice, headers_out, objdevice) def delete_at_update(self, op, delete_at, account, container, obj, request, objdevice): """ Update the expiring objects container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param delete_at: scheduled delete in UNIX seconds, int :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request driving the update :param objdevice: device name that the object is in """ if config_true_value( request.headers.get('x-backend-replication', 'f')): return delete_at = normalize_delete_at_timestamp(delete_at) updates = [(None, None)] partition = None hosts = contdevices = [None] headers_in = request.headers headers_out = HeaderKeyDict({ 'x-timestamp': headers_in['x-timestamp'], 'x-trans-id': headers_in.get('x-trans-id', '-'), 'referer': request.as_referer()}) if op != 'DELETE': delete_at_container = headers_in.get('X-Delete-At-Container', None) if not delete_at_container: self.logger.warning( 'X-Delete-At-Container header must be specified for ' 'expiring objects background %s to work properly. Making ' 'best guess as to the container name for now.' % op) # TODO(gholt): In a future release, change the above warning to # a raised exception and remove the guess code below. delete_at_container = ( int(delete_at) / self.expiring_objects_container_divisor * self.expiring_objects_container_divisor) partition = headers_in.get('X-Delete-At-Partition', None) hosts = headers_in.get('X-Delete-At-Host', '') contdevices = headers_in.get('X-Delete-At-Device', '') updates = [upd for upd in zip((h.strip() for h in hosts.split(',')), (c.strip() for c in contdevices.split(','))) if all(upd) and partition] if not updates: updates = [(None, None)] headers_out['x-size'] = '0' headers_out['x-content-type'] = 'text/plain' headers_out['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e' else: # DELETEs of old expiration data have no way of knowing what the # old X-Delete-At-Container was at the time of the initial setting # of the data, so a best guess is made here. # Worst case is a DELETE is issued now for something that doesn't # exist there and the original data is left where it is, where # it will be ignored when the expirer eventually tries to issue the # object DELETE later since the X-Delete-At value won't match up. delete_at_container = str( int(delete_at) / self.expiring_objects_container_divisor * self.expiring_objects_container_divisor) delete_at_container = normalize_delete_at_timestamp( delete_at_container) for host, contdevice in updates: self.async_update( op, self.expiring_objects_account, delete_at_container, '%s-%s/%s/%s' % (delete_at, account, container, obj), host, partition, contdevice, headers_out, objdevice) @public @timing_stats() def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = orig_metadata.get('X-Timestamp', '0') if orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) metadata = {'X-Timestamp': request.headers['x-timestamp']} metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device) disk_file.write_metadata(metadata) return HTTPAccepted(request=request) @public @timing_stats() def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') error_response = check_object_creation(request, obj) if error_response: return error_response new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} # Checks for If-None-Match if request.if_none_match is not None and orig_metadata: if '*' in request.if_none_match: # File exists already so return 412 return HTTPPreconditionFailed(request=request) if orig_metadata.get('ETag') in request.if_none_match: # The current ETag matches, so return 412 return HTTPPreconditionFailed(request=request) orig_timestamp = orig_metadata.get('X-Timestamp') if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 try: with disk_file.create(size=fsize) as writer: upload_size = 0 def timeout_reader(): with ChunkReadTimeout(self.client_timeout): return request.environ['wsgi.input'].read( self.network_chunk_size) try: for chunk in iter(lambda: timeout_reader(), ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time except ChunkReadTimeout: return HTTPRequestTimeout(request=request) if upload_size: self.logger.transfer_rate( 'PUT.' + device + '.timing', elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { 'X-Timestamp': request.headers['x-timestamp'], 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in ( request.headers.get('X-Backend-Replication-Headers') or self.allowed_headers): if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) except DiskFileNoSpace: return HTTPInsufficientStorage(drive=device, request=request) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update( 'PUT', new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update( 'DELETE', orig_delete_at, account, container, obj, request, device) self.container_update( 'PUT', account, container, obj, request, HeaderKeyDict({ 'x-size': metadata['Content-Length'], 'x-content-type': metadata['Content-Type'], 'x-timestamp': metadata['X-Timestamp'], 'x-etag': metadata['ETag']}), device) return HTTPCreated(request=request, etag=etag) @public @timing_stats() def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) file_x_ts = metadata['X-Timestamp'] file_x_ts_flt = float(file_x_ts) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(file_x_ts_flt) response.content_length = obj_size try: response.content_encoding = metadata[ 'Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts resp = request.get_response(response) except (DiskFileNotExist, DiskFileQuarantined): resp = HTTPNotFound(request=request, conditional_response=True) return resp @public @timing_stats(sample_rate=0.8) def HEAD(self, request): """Handle HTTP HEAD requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request, conditional_response=True) response = Response(request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] ts = metadata['X-Timestamp'] response.last_modified = math.ceil(float(ts)) # Needed for container sync feature response.headers['X-Timestamp'] = ts response.content_length = int(metadata['Content-Length']) try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass return response @public @timing_stats() def DELETE(self, request): """Handle HTTP DELETE requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileExpired as e: orig_timestamp = e.timestamp orig_metadata = e.metadata response_class = HTTPNotFound except DiskFileDeleted as e: orig_timestamp = e.timestamp orig_metadata = {} response_class = HTTPNotFound except (DiskFileNotExist, DiskFileQuarantined): orig_timestamp = 0 orig_metadata = {} response_class = HTTPNotFound else: orig_timestamp = orig_metadata.get('X-Timestamp', 0) if orig_timestamp < request.headers['x-timestamp']: response_class = HTTPNoContent else: response_class = HTTPConflict orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) try: req_if_delete_at_val = request.headers['x-if-delete-at'] req_if_delete_at = int(req_if_delete_at_val) except KeyError: pass except ValueError: return HTTPBadRequest( request=request, body='Bad X-If-Delete-At header value') else: if orig_delete_at != req_if_delete_at: return HTTPPreconditionFailed( request=request, body='X-If-Delete-At and X-Delete-At do not match') if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device) req_timestamp = request.headers['X-Timestamp'] if orig_timestamp < req_timestamp: disk_file.delete(req_timestamp) self.container_update( 'DELETE', account, container, obj, request, HeaderKeyDict({'x-timestamp': req_timestamp}), device) return response_class(request=request) @public @replication @timing_stats(sample_rate=0.1) def REPLICATE(self, request): """ Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. """ device, partition, suffix = split_and_validate_path( request, 2, 3, True) try: hashes = self._diskfile_mgr.get_hashes(device, partition, suffix) except DiskFileDeviceUnavailable: resp = HTTPInsufficientStorage(drive=device, request=request) else: resp = Response(body=pickle.dumps(hashes)) return resp @public @replication @timing_stats(sample_rate=0.1) def REPLICATION(self, request): return Response(app_iter=ssync_receiver.Receiver(self, request)()) def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception(_( 'ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = get_log_line(req, res, trans_time, '') if req.method in ('REPLICATE', 'REPLICATION') or \ 'X-Backend-Replication' in req.headers: self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time if slow > 0: sleep(slow) return res(env, start_response)
class ObjectController(BaseStorageServer): """Implements the WSGI application for the Swift Object Server.""" server_type = 'object-server' def __init__(self, conf, logger=None): """ Creates a new WSGI application for the Swift Object Server. An example configuration is given at <source-dir>/etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ super(ObjectController, self).__init__(conf) self.logger = logger or get_logger(conf, log_route='object-server') self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) self.network_chunk_size = int(conf.get('network_chunk_size', 65536)) self.log_requests = config_true_value(conf.get('log_requests', 'true')) self.max_upload_time = int(conf.get('max_upload_time', 86400)) self.slow = int(conf.get('slow', 0)) self.keep_cache_private = \ config_true_value(conf.get('keep_cache_private', 'false')) default_allowed_headers = ''' content-disposition, content-encoding, x-delete-at, x-object-manifest, x-static-large-object, ''' extra_allowed_headers = [ header.strip().lower() for header in conf.get( 'allowed_headers', default_allowed_headers).split(',') if header.strip() ] self.allowed_headers = set() for header in extra_allowed_headers: if header not in DATAFILE_SYSTEM_META: self.allowed_headers.add(header) self.auto_create_account_prefix = \ conf.get('auto_create_account_prefix') or '.' self.expiring_objects_account = self.auto_create_account_prefix + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) # Initialization was successful, so now apply the network chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because the primary motivation for this is to optimize how data # is written back to the proxy server, we could use the value from the # disk_chunk_size parameter. However, it affects all created sockets # using this class so we have chosen to tie it to the # network_chunk_size parameter value instead. socket._fileobject.default_bufsize = self.network_chunk_size # Provide further setup specific to an object server implementation. self.setup(conf) def setup(self, conf): """ Implementation specific setup. This method is called at the very end by the constructor to allow a specific implementation to modify existing attributes or add its own attributes. :param conf: WSGI configuration parameter """ # Common on-disk hierarchy shared across account, container and object # servers. self._diskfile_mgr = DiskFileManager(conf, self.logger) # This is populated by global_conf_callback way below as the semaphore # is shared by all workers. if 'replication_semaphore' in conf: # The value was put in a list so it could get past paste self.replication_semaphore = conf['replication_semaphore'][0] else: self.replication_semaphore = None self.replication_failure_threshold = int( conf.get('replication_failure_threshold') or 100) self.replication_failure_ratio = float( conf.get('replication_failure_ratio') or 1.0) def get_diskfile(self, device, partition, account, container, obj, policy_idx, **kwargs): """ Utility method for instantiating a DiskFile object supporting a given REST API. An implementation of the object server that wants to use a different DiskFile class would simply over-ride this method to provide that behavior. """ return self._diskfile_mgr.get_diskfile(device, partition, account, container, obj, policy_idx, **kwargs) def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice, policy_index): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in :param policy_index: the associated storage policy index """ headers_out['user-agent'] = 'object-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error( _('ERROR Container update failed ' '(saving for async update later): %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), { 'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice }) except (Exception, Timeout): self.logger.exception( _('ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s (saving for async update later)' ), { 'ip': ip, 'port': port, 'dev': contdevice }) data = { 'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out } timestamp = headers_out['x-timestamp'] self._diskfile_mgr.pickle_async_update(objdevice, account, container, obj, data, timestamp, policy_index) def container_update(self, op, account, container, obj, request, headers_out, objdevice, policy_idx): """ Update the container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request object driving the update :param headers_out: dictionary of headers to send in the container request(s) :param objdevice: device name that the object is in """ headers_in = request.headers conthosts = [ h.strip() for h in headers_in.get('X-Container-Host', '').split(',') ] contdevices = [ d.strip() for d in headers_in.get('X-Container-Device', '').split(',') ] contpartition = headers_in.get('X-Container-Partition', '') if len(conthosts) != len(contdevices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. self.logger.error( _('ERROR Container update failed: different ' 'numbers of hosts and devices in request: ' '"%s" vs "%s"') % (headers_in.get('X-Container-Host', ''), headers_in.get('X-Container-Device', ''))) return if contpartition: updates = zip(conthosts, contdevices) else: updates = [] headers_out['x-trans-id'] = headers_in.get('x-trans-id', '-') headers_out['referer'] = request.as_referer() headers_out['X-Backend-Storage-Policy-Index'] = policy_idx for conthost, contdevice in updates: self.async_update(op, account, container, obj, conthost, contpartition, contdevice, headers_out, objdevice, policy_idx) def delete_at_update(self, op, delete_at, account, container, obj, request, objdevice, policy_index): """ Update the expiring objects container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param delete_at: scheduled delete in UNIX seconds, int :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request driving the update :param objdevice: device name that the object is in :param policy_index: the policy index to be used for tmp dir """ if config_true_value(request.headers.get('x-backend-replication', 'f')): return delete_at = normalize_delete_at_timestamp(delete_at) updates = [(None, None)] partition = None hosts = contdevices = [None] headers_in = request.headers headers_out = HeaderKeyDict({ # system accounts are always Policy-0 'X-Backend-Storage-Policy-Index': 0, 'x-timestamp': request.timestamp.internal, 'x-trans-id': headers_in.get('x-trans-id', '-'), 'referer': request.as_referer() }) if op != 'DELETE': delete_at_container = headers_in.get('X-Delete-At-Container', None) if not delete_at_container: self.logger.warning( 'X-Delete-At-Container header must be specified for ' 'expiring objects background %s to work properly. Making ' 'best guess as to the container name for now.' % op) # TODO(gholt): In a future release, change the above warning to # a raised exception and remove the guess code below. delete_at_container = get_expirer_container( delete_at, self.expiring_objects_container_divisor, account, container, obj) partition = headers_in.get('X-Delete-At-Partition', None) hosts = headers_in.get('X-Delete-At-Host', '') contdevices = headers_in.get('X-Delete-At-Device', '') updates = [ upd for upd in zip((h.strip() for h in hosts.split(',')), ( c.strip() for c in contdevices.split(','))) if all(upd) and partition ] if not updates: updates = [(None, None)] headers_out['x-size'] = '0' headers_out['x-content-type'] = 'text/plain' headers_out['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e' else: # DELETEs of old expiration data have no way of knowing what the # old X-Delete-At-Container was at the time of the initial setting # of the data, so a best guess is made here. # Worst case is a DELETE is issued now for something that doesn't # exist there and the original data is left where it is, where # it will be ignored when the expirer eventually tries to issue the # object DELETE later since the X-Delete-At value won't match up. delete_at_container = get_expirer_container( delete_at, self.expiring_objects_container_divisor, account, container, obj) delete_at_container = normalize_delete_at_timestamp( delete_at_container) for host, contdevice in updates: self.async_update( op, self.expiring_objects_account, delete_at_container, '%s-%s/%s/%s' % (delete_at, account, container, obj), host, partition, contdevice, headers_out, objdevice, policy_index) @public @timing_stats() def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: return HTTPConflict( request=request, headers={'X-Backend-Timestamp': orig_timestamp.internal}) metadata = {'X-Timestamp': req_timestamp.internal} metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device, policy_idx) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy_idx) try: disk_file.write_metadata(metadata) except (DiskFileXattrNotSupported, DiskFileNoSpace): return HTTPInsufficientStorage(drive=device, request=request) return HTTPAccepted(request=request) @public @timing_stats() def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) error_response = check_object_creation(request, obj) if error_response: return error_response new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} # Checks for If-None-Match if request.if_none_match is not None and orig_metadata: if '*' in request.if_none_match: # File exists already so return 412 return HTTPPreconditionFailed(request=request) if orig_metadata.get('ETag') in request.if_none_match: # The current ETag matches, so return 412 return HTTPPreconditionFailed(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: return HTTPConflict( request=request, headers={'X-Backend-Timestamp': orig_timestamp.internal}) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 try: with disk_file.create(size=fsize) as writer: upload_size = 0 def timeout_reader(): with ChunkReadTimeout(self.client_timeout): return request.environ['wsgi.input'].read( self.network_chunk_size) try: for chunk in iter(lambda: timeout_reader(), ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time except ChunkReadTimeout: return HTTPRequestTimeout(request=request) if upload_size: self.logger.transfer_rate('PUT.' + device + '.timing', elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { 'X-Timestamp': request.timestamp.internal, 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.iteritems() if is_sys_or_user_meta('object', val[0])) headers_to_copy = (request.headers.get( 'X-Backend-Replication-Headers', '').split() + list(self.allowed_headers)) for header_key in headers_to_copy: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) except (DiskFileXattrNotSupported, DiskFileNoSpace): return HTTPInsufficientStorage(drive=device, request=request) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device, policy_idx) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy_idx) self.container_update( 'PUT', account, container, obj, request, HeaderKeyDict({ 'x-size': metadata['Content-Length'], 'x-content-type': metadata['Content-Type'], 'x-timestamp': metadata['X-Timestamp'], 'x-etag': metadata['ETag'] }), device, policy_idx) return HTTPCreated(request=request, etag=etag) @public @timing_stats() def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) file_x_ts = Timestamp(metadata['X-Timestamp']) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_sys_or_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(float(file_x_ts)) response.content_length = obj_size try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts.normal response.headers['X-Backend-Timestamp'] = file_x_ts.internal resp = request.get_response(response) except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): headers['X-Backend-Timestamp'] = e.timestamp.internal resp = HTTPNotFound(request=request, headers=headers, conditional_response=True) return resp @public @timing_stats(sample_rate=0.8) def HEAD(self, request): """Handle HTTP HEAD requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): headers['X-Backend-Timestamp'] = e.timestamp.internal return HTTPNotFound(request=request, headers=headers, conditional_response=True) response = Response(request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_sys_or_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] ts = Timestamp(metadata['X-Timestamp']) response.last_modified = math.ceil(float(ts)) # Needed for container sync feature response.headers['X-Timestamp'] = ts.normal response.headers['X-Backend-Timestamp'] = ts.internal response.content_length = int(metadata['Content-Length']) try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass return response @public @timing_stats() def DELETE(self, request): """Handle HTTP DELETE requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except DiskFileExpired as e: orig_timestamp = e.timestamp orig_metadata = e.metadata response_class = HTTPNotFound except DiskFileDeleted as e: orig_timestamp = e.timestamp orig_metadata = {} response_class = HTTPNotFound except (DiskFileNotExist, DiskFileQuarantined): orig_timestamp = 0 orig_metadata = {} response_class = HTTPNotFound else: orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp < req_timestamp: response_class = HTTPNoContent else: response_class = HTTPConflict response_timestamp = max(orig_timestamp, req_timestamp) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) try: req_if_delete_at_val = request.headers['x-if-delete-at'] req_if_delete_at = int(req_if_delete_at_val) except KeyError: pass except ValueError: return HTTPBadRequest(request=request, body='Bad X-If-Delete-At header value') else: # request includes x-if-delete-at; we must not place a tombstone # if we can not verify the x-if-delete-at time if not orig_timestamp: # no object found at all return HTTPNotFound() if orig_delete_at != req_if_delete_at: return HTTPPreconditionFailed( request=request, body='X-If-Delete-At and X-Delete-At do not match') else: # differentiate success from no object at all response_class = HTTPNoContent if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy_idx) if orig_timestamp < req_timestamp: disk_file.delete(req_timestamp) self.container_update( 'DELETE', account, container, obj, request, HeaderKeyDict({'x-timestamp': req_timestamp.internal}), device, policy_idx) return response_class( request=request, headers={'X-Backend-Timestamp': response_timestamp.internal}) @public @replication @timing_stats(sample_rate=0.1) def REPLICATE(self, request): """ Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. """ device, partition, suffix, policy_idx = \ get_name_and_placement(request, 2, 3, True) try: hashes = self._diskfile_mgr.get_hashes(device, partition, suffix, policy_idx) except DiskFileDeviceUnavailable: resp = HTTPInsufficientStorage(drive=device, request=request) else: resp = Response(body=pickle.dumps(hashes)) return resp @public @replication @timing_stats(sample_rate=0.1) def REPLICATION(self, request): return Response(app_iter=ssync_receiver.Receiver(self, request)()) def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: if req.method not in self.allowed_methods: raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: method = getattr(self, req.method) res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception( _('ERROR __call__ error with %(method)s' ' %(path)s '), { 'method': req.method, 'path': req.path }) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = get_log_line(req, res, trans_time, '') if req.method in ('REPLICATE', 'REPLICATION') or \ 'X-Backend-Replication' in req.headers: self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time if slow > 0: sleep(slow) # To be able to zero-copy send the object, we need a few things. # First, we have to be responding successfully to a GET, or else we're # not sending the object. Second, we have to be able to extract the # socket file descriptor from the WSGI input object. Third, the # diskfile has to support zero-copy send. # # There's a good chance that this could work for 206 responses too, # but the common case is sending the whole object, so we'll start # there. if req.method == 'GET' and res.status_int == 200 and \ isinstance(env['wsgi.input'], wsgi.Input): app_iter = getattr(res, 'app_iter', None) checker = getattr(app_iter, 'can_zero_copy_send', None) if checker and checker(): # For any kind of zero-copy thing like sendfile or splice, we # need the file descriptor. Eventlet doesn't provide a clean # way of getting that, so we resort to this. wsock = env['wsgi.input'].get_socket() wsockfd = wsock.fileno() # Don't call zero_copy_send() until after we force the HTTP # headers out of Eventlet and into the socket. def zero_copy_iter(): # If possible, set TCP_CORK so that headers don't # immediately go on the wire, but instead, wait for some # response body to make the TCP frames as large as # possible (and hence as few packets as possible). # # On non-Linux systems, we might consider TCP_NODELAY, but # since the only known zero-copy-capable diskfile uses # Linux-specific syscalls, we'll defer that work until # someone needs it. if hasattr(socket, 'TCP_CORK'): wsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_CORK, 1) yield EventletPlungerString() try: app_iter.zero_copy_send(wsockfd) except Exception: self.logger.exception("zero_copy_send() blew up") raise yield '' # Get headers ready to go out res(env, start_response) return zero_copy_iter() else: return res(env, start_response) else: return res(env, start_response)
class ObjectController(object): """Implements the WSGI application for the Swift Object Server.""" def __init__(self, conf, logger=None): """ Creates a new WSGI application for the Swift Object Server. An example configuration is given at <source-dir>/etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ self.logger = logger or get_logger(conf, log_route='object-server') self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) self.network_chunk_size = int(conf.get('network_chunk_size', 65536)) self.log_requests = config_true_value(conf.get('log_requests', 'true')) self.max_upload_time = int(conf.get('max_upload_time', 86400)) self.slow = int(conf.get('slow', 0)) self.keep_cache_private = \ config_true_value(conf.get('keep_cache_private', 'false')) replication_server = conf.get('replication_server', None) if replication_server is not None: replication_server = config_true_value(replication_server) self.replication_server = replication_server default_allowed_headers = ''' content-disposition, content-encoding, x-delete-at, x-object-manifest, x-static-large-object, ''' extra_allowed_headers = [ header.strip().lower() for header in conf.get( 'allowed_headers', default_allowed_headers).split(',') if header.strip() ] self.allowed_headers = set() for header in extra_allowed_headers: if header not in DATAFILE_SYSTEM_META: self.allowed_headers.add(header) self.expiring_objects_account = \ (conf.get('auto_create_account_prefix') or '.') + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) # Initialization was successful, so now apply the network chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because the primary motivation for this is to optimize how data # is written back to the proxy server, we could use the value from the # disk_chunk_size parameter. However, it affects all created sockets # using this class so we have chosen to tie it to the # network_chunk_size parameter value instead. socket._fileobject.default_bufsize = self.network_chunk_size # Provide further setup sepecific to an object server implemenation. self.setup(conf) def setup(self, conf): """ Implementation specific setup. This method is called at the very end by the constructor to allow a specific implementation to modify existing attributes or add its own attributes. :param conf: WSGI configuration parameter """ # Common on-disk hierarchy shared across account, container and object # servers. self._diskfile_mgr = DiskFileManager(conf, self.logger) # This is populated by global_conf_callback way below as the semaphore # is shared by all workers. if 'replication_semaphore' in conf: # The value was put in a list so it could get past paste self.replication_semaphore = conf['replication_semaphore'][0] else: self.replication_semaphore = None self.replication_failure_threshold = int( conf.get('replication_failure_threshold') or 100) self.replication_failure_ratio = float( conf.get('replication_failure_ratio') or 1.0) def get_diskfile(self, device, partition, account, container, obj, **kwargs): """ Utility method for instantiating a DiskFile object supporting a given REST API. An implementation of the object server that wants to use a different DiskFile class would simply over-ride this method to provide that behavior. """ return self._diskfile_mgr.get_diskfile( device, partition, account, container, obj, **kwargs) def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ headers_out['user-agent'] = 'obj-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error(_( 'ERROR Container update failed ' '(saving for async update later): %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), {'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice}) except (Exception, Timeout): self.logger.exception(_( 'ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s (saving for async update later)'), {'ip': ip, 'port': port, 'dev': contdevice}) data = {'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out} timestamp = headers_out['x-timestamp'] self._diskfile_mgr.pickle_async_update(objdevice, account, container, obj, data, timestamp) def container_update(self, op, account, container, obj, request, headers_out, objdevice): """ Update the container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request object driving the update :param headers_out: dictionary of headers to send in the container request(s) :param objdevice: device name that the object is in """ headers_in = request.headers conthosts = [h.strip() for h in headers_in.get('X-Container-Host', '').split(',')] contdevices = [d.strip() for d in headers_in.get('X-Container-Device', '').split(',')] contpartition = headers_in.get('X-Container-Partition', '') if len(conthosts) != len(contdevices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. self.logger.error(_('ERROR Container update failed: different ' 'numbers of hosts and devices in request: ' '"%s" vs "%s"') % (headers_in.get('X-Container-Host', ''), headers_in.get('X-Container-Device', ''))) return if contpartition: updates = zip(conthosts, contdevices) else: updates = [] headers_out['x-trans-id'] = headers_in.get('x-trans-id', '-') headers_out['referer'] = request.as_referer() for conthost, contdevice in updates: self.async_update(op, account, container, obj, conthost, contpartition, contdevice, headers_out, objdevice) def delete_at_update(self, op, delete_at, account, container, obj, request, objdevice): """ Update the expiring objects container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param delete_at: scheduled delete in UNIX seconds, int :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request driving the update :param objdevice: device name that the object is in """ if config_true_value( request.headers.get('x-backend-replication', 'f')): return delete_at = normalize_delete_at_timestamp(delete_at) updates = [(None, None)] partition = None hosts = contdevices = [None] headers_in = request.headers headers_out = HeaderKeyDict({ 'x-timestamp': headers_in['x-timestamp'], 'x-trans-id': headers_in.get('x-trans-id', '-'), 'referer': request.as_referer()}) if op != 'DELETE': delete_at_container = headers_in.get('X-Delete-At-Container', None) if not delete_at_container: self.logger.warning( 'X-Delete-At-Container header must be specified for ' 'expiring objects background %s to work properly. Making ' 'best guess as to the container name for now.' % op) # TODO(gholt): In a future release, change the above warning to # a raised exception and remove the guess code below. delete_at_container = ( int(delete_at) / self.expiring_objects_container_divisor * self.expiring_objects_container_divisor) partition = headers_in.get('X-Delete-At-Partition', None) hosts = headers_in.get('X-Delete-At-Host', '') contdevices = headers_in.get('X-Delete-At-Device', '') updates = [upd for upd in zip((h.strip() for h in hosts.split(',')), (c.strip() for c in contdevices.split(','))) if all(upd) and partition] if not updates: updates = [(None, None)] headers_out['x-size'] = '0' headers_out['x-content-type'] = 'text/plain' headers_out['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e' else: # DELETEs of old expiration data have no way of knowing what the # old X-Delete-At-Container was at the time of the initial setting # of the data, so a best guess is made here. # Worst case is a DELETE is issued now for something that doesn't # exist there and the original data is left where it is, where # it will be ignored when the expirer eventually tries to issue the # object DELETE later since the X-Delete-At value won't match up. delete_at_container = str( int(delete_at) / self.expiring_objects_container_divisor * self.expiring_objects_container_divisor) delete_at_container = normalize_delete_at_timestamp( delete_at_container) for host, contdevice in updates: self.async_update( op, self.expiring_objects_account, delete_at_container, '%s-%s/%s/%s' % (delete_at, account, container, obj), host, partition, contdevice, headers_out, objdevice) @public @timing_stats() def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = orig_metadata.get('X-Timestamp', '0') if orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) metadata = {'X-Timestamp': request.headers['x-timestamp']} metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device) disk_file.write_metadata(metadata) return HTTPAccepted(request=request) @public @timing_stats() def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) my_debug("local variables in PUT()",locals()) my_debug("request.path", request.path) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') error_response = check_object_creation(request, obj) if error_response: return error_response new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} # Checks for If-None-Match if request.if_none_match is not None and orig_metadata: if '*' in request.if_none_match: # File exists already so return 412 return HTTPPreconditionFailed(request=request) if orig_metadata.get('ETag') in request.if_none_match: # The current ETag matches, so return 412 return HTTPPreconditionFailed(request=request) orig_timestamp = orig_metadata.get('X-Timestamp') if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 try: with disk_file.create(size=fsize) as writer: upload_size = 0 def timeout_reader(): with ChunkReadTimeout(self.client_timeout): return request.environ['wsgi.input'].read( self.network_chunk_size) try: for chunk in iter(lambda: timeout_reader(), ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time except ChunkReadTimeout: return HTTPRequestTimeout(request=request) if upload_size: self.logger.transfer_rate( 'PUT.' + device + '.timing', elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { 'X-Timestamp': request.headers['x-timestamp'], 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in ( request.headers.get('X-Backend-Replication-Headers') or self.allowed_headers): if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) except DiskFileNoSpace: return HTTPInsufficientStorage(drive=device, request=request) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update( 'PUT', new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update( 'DELETE', orig_delete_at, account, container, obj, request, device) self.container_update( 'PUT', account, container, obj, request, HeaderKeyDict({ 'x-size': metadata['Content-Length'], 'x-content-type': metadata['Content-Type'], 'x-timestamp': metadata['X-Timestamp'], 'x-etag': metadata['ETag']}), device) return HTTPCreated(request=request, etag=etag) def tmp_disk_file(self, request=None, device=None, partition=None,container=None, obj=None, data=None, account=None): try: disk_file = self.get_diskfile( device, partition, account, container, obj) with disk_file.create(size=len(data)) as writer: #writer = disk_file.create(size=len(data)) my_debug( "diskfile.create", writer) writer.write(data) etag = md5() etag.update(data) etag = etag.hexdigest() metadata = { 'X-Timestamp': request.headers['X-Timestamp'], 'Content-Type': 'application/json', 'Content-Length': len(data), 'X-Delete-At':time.time() + 1000, 'ETag': etag, } writer.put(metadata) return disk_file except Exception as e: my_debug("error creating tmp file", e) return None @public @timing_stats() def GET(self, request): __CBAC__ = True """Handle HTTP GET requests for the Swift Object Server.""" my_debug('testing', '@GET()') my_debug('#request headers', request.headers) #print my_debug device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) my_debug("locals()",locals()) my_debug("original Diskfile", disk_file) try: with disk_file.open(): metadata = disk_file.get_metadata() my_debug("metadata", metadata) obj_size = int(metadata['Content-Length']) file_x_ts = metadata['X-Timestamp'] #json_policy = metadata['JSON-Policy'] file_x_ts_flt = float(file_x_ts) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) ''' if policy file is present & user_label is present apply JSONAC Additional thing to have: 1. Check if policy from metadata is valid json 2. Check if user_clearance is present is headers. Read keystone info. 3. Pass JSONPath as headers 4. ''' if __CBAC__: my_debug("----CBAC Starts here-----", "***********") disk_file_iter = disk_file.reader(keep_cache=keep_cache) original_data = "" for chunk in iter(disk_file_iter): original_data += chunk userRoles = request.headers['X-Roles'] json_policy = metadata['X-Object-Meta-Jsonpolicy'] \ if metadata.has_key('X-Object-Meta-Jsonpolicy') else None user_labels = metadata['X-Object-Meta-Userlabels'] \ if metadata.has_key('X-Object-Meta-Userlabels') else None object_labels = metadata['X-Object-Meta-Objectlabels'] \ if metadata.has_key('X-Object-Meta-Objectlabels') else None object_labelling = metadata['X-Object-Meta-Jsonlabelling'] \ if metadata.has_key('X-Object-Meta-Jsonlabelling') else None if json_policy and user_labels and object_labels : cbac_policy = {} cbac_policy['user_labels'] = json.loads(user_labels) cbac_policy['object_labels'] = json.loads(object_labels) cbac_policy['policy'] = json.loads(json_policy) #user_clearance = ['manager'] user_clearance = userRoles jsonpath = "/" filtered_content = "" my_debug("json_policy is", json_policy) my_debug("original data is ", original_data) #try: if json_policy : if json_policy and user_clearance and jsonpath and object_labelling: '''filtered_content = ContentFilter(content_str=original_data, labeling_policy_str=object_labelling, user_clearance=user_clearance, query=jsonpath, cbac_policy=cbac_policy).apply()''' filtered_content = "testing" my_debug("#filtered content is ", filtered_content) #else: #my_debug("#content_filter not working", True) #except Exception as e: #else: # my_debug("Exception with content filtering {}".format("e"), True) '''end of content -filter ''' tmp_file = self.tmp_disk_file( request=request, device=device, partition=partition, container=container, obj="tmp", data=filtered_content, account=account) tmp_file = self.get_diskfile( device, partition, account, container, "tmp") my_debug("tmp_file is", tmp_file) try: with tmp_file.open(): tmp_metadata = tmp_file.get_metadata() my_debug("with tmp_file.open()", tmp_file.reader(keep_cache=keep_cache)) response = Response( app_iter=tmp_file.reader(keep_cache=keep_cache), request=request, conditional_response=True) response.headers['Content-Type'] = tmp_metadata.get( 'Content-Type', 'application/octet-stream') response.content_length = len(filtered_content) for key, value in tmp_metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = tmp_metadata['ETag'] #response.last_modified = math.ceil(file_x_ts_flt) #response.content_length = obj_size my_debug("get_response", "test") resp = request.get_response(response) my_debug( "response", resp ) my_debug("rsponse.__dict__", resp.__dict__) except (DiskFileNotExist, DiskFileQuarantined): my_debug("tmp file is not found", "test") resp = HTTPNotFound(request=request, conditional_response=True) my_debug("final resp object", resp) return resp disk_file.open() response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), # instead of app_iter which reads from the file, content is given in body #body="Yes, you are smart!\n", request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(file_x_ts_flt) response.content_length = obj_size #response.content_length = response.headers['Content-Length'] try: response.content_encoding = metadata[ 'Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts resp = request.get_response(response) except (DiskFileNotExist, DiskFileQuarantined): resp = HTTPNotFound(request=request, conditional_response=True) my_debug("resp object without cbac", resp.__dict__) return resp @public @timing_stats(sample_rate=0.8) def HEAD(self, request): """Handle HTTP HEAD requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request, conditional_response=True) response = Response(request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] ts = metadata['X-Timestamp'] response.last_modified = math.ceil(float(ts)) # Needed for container sync feature response.headers['X-Timestamp'] = ts response.content_length = int(metadata['Content-Length']) try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass return response @public @timing_stats() def DELETE(self, request): """Handle HTTP DELETE requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileExpired as e: orig_timestamp = e.timestamp orig_metadata = e.metadata response_class = HTTPNotFound except DiskFileDeleted as e: orig_timestamp = e.timestamp orig_metadata = {} response_class = HTTPNotFound except (DiskFileNotExist, DiskFileQuarantined): orig_timestamp = 0 orig_metadata = {} response_class = HTTPNotFound else: orig_timestamp = orig_metadata.get('X-Timestamp', 0) if orig_timestamp < request.headers['x-timestamp']: response_class = HTTPNoContent else: response_class = HTTPConflict orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) try: req_if_delete_at_val = request.headers['x-if-delete-at'] req_if_delete_at = int(req_if_delete_at_val) except KeyError: pass except ValueError: return HTTPBadRequest( request=request, body='Bad X-If-Delete-At header value') else: if orig_delete_at != req_if_delete_at: return HTTPPreconditionFailed( request=request, body='X-If-Delete-At and X-Delete-At do not match') if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device) req_timestamp = request.headers['X-Timestamp'] if orig_timestamp < req_timestamp: disk_file.delete(req_timestamp) self.container_update( 'DELETE', account, container, obj, request, HeaderKeyDict({'x-timestamp': req_timestamp}), device) return response_class(request=request) @public @replication @timing_stats(sample_rate=0.1) def REPLICATE(self, request): """ Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. """ device, partition, suffix = split_and_validate_path( request, 2, 3, True) try: hashes = self._diskfile_mgr.get_hashes(device, partition, suffix) except DiskFileDeviceUnavailable: resp = HTTPInsufficientStorage(drive=device, request=request) else: resp = Response(body=pickle.dumps(hashes)) return resp @public @replication @timing_stats(sample_rate=0.1) def REPLICATION(self, request): return Response(app_iter=ssync_receiver.Receiver(self, request)()) def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception(_( 'ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = get_log_line(req, res, trans_time, '') if req.method in ('REPLICATE', 'REPLICATION') or \ 'X-Backend-Replication' in req.headers: self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time if slow > 0: sleep(slow) my_debug('returning from __call__ or objserver', res) my_debug('returnign from __call__ of objserver', start_response) return res(env, start_response)
class ObjectController(BaseStorageServer): """Implements the WSGI application for the Swift Object Server.""" def __init__(self, conf, logger=None): """ Creates a new WSGI application for the Swift Object Server. An example configuration is given at <source-dir>/etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ super(ObjectController, self).__init__(conf) self.logger = logger or get_logger(conf, log_route="object-server") self.node_timeout = int(conf.get("node_timeout", 3)) self.conn_timeout = float(conf.get("conn_timeout", 0.5)) self.client_timeout = int(conf.get("client_timeout", 60)) self.disk_chunk_size = int(conf.get("disk_chunk_size", 65536)) self.network_chunk_size = int(conf.get("network_chunk_size", 65536)) self.log_requests = config_true_value(conf.get("log_requests", "true")) self.max_upload_time = int(conf.get("max_upload_time", 86400)) self.slow = int(conf.get("slow", 0)) self.keep_cache_private = config_true_value(conf.get("keep_cache_private", "false")) default_allowed_headers = """ content-disposition, content-encoding, x-delete-at, x-object-manifest, x-static-large-object, """ extra_allowed_headers = [ header.strip().lower() for header in conf.get("allowed_headers", default_allowed_headers).split(",") if header.strip() ] self.allowed_headers = set() for header in extra_allowed_headers: if header not in DATAFILE_SYSTEM_META: self.allowed_headers.add(header) self.auto_create_account_prefix = conf.get("auto_create_account_prefix") or "." self.expiring_objects_account = self.auto_create_account_prefix + ( conf.get("expiring_objects_account_name") or "expiring_objects" ) self.expiring_objects_container_divisor = int(conf.get("expiring_objects_container_divisor") or 86400) # Initialization was successful, so now apply the network chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because the primary motivation for this is to optimize how data # is written back to the proxy server, we could use the value from the # disk_chunk_size parameter. However, it affects all created sockets # using this class so we have chosen to tie it to the # network_chunk_size parameter value instead. socket._fileobject.default_bufsize = self.network_chunk_size # Provide further setup specific to an object server implementation. self.setup(conf) def setup(self, conf): """ Implementation specific setup. This method is called at the very end by the constructor to allow a specific implementation to modify existing attributes or add its own attributes. :param conf: WSGI configuration parameter """ # Common on-disk hierarchy shared across account, container and object # servers. self._diskfile_mgr = DiskFileManager(conf, self.logger) # This is populated by global_conf_callback way below as the semaphore # is shared by all workers. if "replication_semaphore" in conf: # The value was put in a list so it could get past paste self.replication_semaphore = conf["replication_semaphore"][0] else: self.replication_semaphore = None self.replication_failure_threshold = int(conf.get("replication_failure_threshold") or 100) self.replication_failure_ratio = float(conf.get("replication_failure_ratio") or 1.0) def get_diskfile(self, device, partition, account, container, obj, policy_idx, **kwargs): """ Utility method for instantiating a DiskFile object supporting a given REST API. An implementation of the object server that wants to use a different DiskFile class would simply over-ride this method to provide that behavior. """ return self._diskfile_mgr.get_diskfile(device, partition, account, container, obj, policy_idx, **kwargs) def async_update( self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice, policy_index ): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in :param policy_index: the associated storage policy index """ headers_out["user-agent"] = "object-server %s" % os.getpid() full_path = "/%s/%s/%s" % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(":", 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error( _( "ERROR Container update failed " "(saving for async update later): %(status)d " "response from %(ip)s:%(port)s/%(dev)s" ), {"status": response.status, "ip": ip, "port": port, "dev": contdevice}, ) except (Exception, Timeout): self.logger.exception( _("ERROR container update failed with " "%(ip)s:%(port)s/%(dev)s (saving for async update later)"), {"ip": ip, "port": port, "dev": contdevice}, ) data = {"op": op, "account": account, "container": container, "obj": obj, "headers": headers_out} timestamp = headers_out["x-timestamp"] self._diskfile_mgr.pickle_async_update(objdevice, account, container, obj, data, timestamp, policy_index) def container_update(self, op, account, container, obj, request, headers_out, objdevice, policy_idx): """ Update the container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request object driving the update :param headers_out: dictionary of headers to send in the container request(s) :param objdevice: device name that the object is in """ headers_in = request.headers conthosts = [h.strip() for h in headers_in.get("X-Container-Host", "").split(",")] contdevices = [d.strip() for d in headers_in.get("X-Container-Device", "").split(",")] contpartition = headers_in.get("X-Container-Partition", "") if len(conthosts) != len(contdevices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. self.logger.error( _( "ERROR Container update failed: different " "numbers of hosts and devices in request: " '"%s" vs "%s"' ) % (headers_in.get("X-Container-Host", ""), headers_in.get("X-Container-Device", "")) ) return if contpartition: updates = zip(conthosts, contdevices) else: updates = [] headers_out["x-trans-id"] = headers_in.get("x-trans-id", "-") headers_out["referer"] = request.as_referer() headers_out["X-Backend-Storage-Policy-Index"] = policy_idx for conthost, contdevice in updates: self.async_update( op, account, container, obj, conthost, contpartition, contdevice, headers_out, objdevice, policy_idx ) def delete_at_update(self, op, delete_at, account, container, obj, request, objdevice, policy_index): """ Update the expiring objects container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param delete_at: scheduled delete in UNIX seconds, int :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request driving the update :param objdevice: device name that the object is in :param policy_index: the policy index to be used for tmp dir """ if config_true_value(request.headers.get("x-backend-replication", "f")): return delete_at = normalize_delete_at_timestamp(delete_at) updates = [(None, None)] partition = None hosts = contdevices = [None] headers_in = request.headers headers_out = HeaderKeyDict( { # system accounts are always Policy-0 "X-Backend-Storage-Policy-Index": 0, "x-timestamp": request.timestamp.internal, "x-trans-id": headers_in.get("x-trans-id", "-"), "referer": request.as_referer(), } ) if op != "DELETE": delete_at_container = headers_in.get("X-Delete-At-Container", None) if not delete_at_container: self.logger.warning( "X-Delete-At-Container header must be specified for " "expiring objects background %s to work properly. Making " "best guess as to the container name for now." % op ) # TODO(gholt): In a future release, change the above warning to # a raised exception and remove the guess code below. delete_at_container = get_expirer_container( delete_at, self.expiring_objects_container_divisor, account, container, obj ) partition = headers_in.get("X-Delete-At-Partition", None) hosts = headers_in.get("X-Delete-At-Host", "") contdevices = headers_in.get("X-Delete-At-Device", "") updates = [ upd for upd in zip((h.strip() for h in hosts.split(",")), (c.strip() for c in contdevices.split(","))) if all(upd) and partition ] if not updates: updates = [(None, None)] headers_out["x-size"] = "0" headers_out["x-content-type"] = "text/plain" headers_out["x-etag"] = "d41d8cd98f00b204e9800998ecf8427e" else: # DELETEs of old expiration data have no way of knowing what the # old X-Delete-At-Container was at the time of the initial setting # of the data, so a best guess is made here. # Worst case is a DELETE is issued now for something that doesn't # exist there and the original data is left where it is, where # it will be ignored when the expirer eventually tries to issue the # object DELETE later since the X-Delete-At value won't match up. delete_at_container = get_expirer_container( delete_at, self.expiring_objects_container_divisor, account, container, obj ) delete_at_container = normalize_delete_at_timestamp(delete_at_container) for host, contdevice in updates: self.async_update( op, self.expiring_objects_account, delete_at_container, "%s-%s/%s/%s" % (delete_at, account, container, obj), host, partition, contdevice, headers_out, objdevice, policy_index, ) @public @timing_stats() def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) new_delete_at = int(request.headers.get("X-Delete-At") or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body="X-Delete-At in past", request=request, content_type="text/plain") try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = Timestamp(orig_metadata.get("X-Timestamp", 0)) if orig_timestamp >= req_timestamp: return HTTPConflict(request=request, headers={"X-Backend-Timestamp": orig_timestamp.internal}) metadata = {"X-Timestamp": req_timestamp.internal} metadata.update(val for val in request.headers.iteritems() if is_user_meta("object", val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get("X-Delete-At") or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update("PUT", new_delete_at, account, container, obj, request, device, policy_idx) if orig_delete_at: self.delete_at_update("DELETE", orig_delete_at, account, container, obj, request, device, policy_idx) try: disk_file.write_metadata(metadata) except (DiskFileXattrNotSupported, DiskFileNoSpace): return HTTPInsufficientStorage(drive=device, request=request) return HTTPAccepted(request=request) @public @timing_stats() def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) error_response = check_object_creation(request, obj) if error_response: return error_response new_delete_at = int(request.headers.get("X-Delete-At") or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body="X-Delete-At in past", request=request, content_type="text/plain") try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type="text/plain") try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} # Checks for If-None-Match if request.if_none_match is not None and orig_metadata: if "*" in request.if_none_match: # File exists already so return 412 return HTTPPreconditionFailed(request=request) if orig_metadata.get("ETag") in request.if_none_match: # The current ETag matches, so return 412 return HTTPPreconditionFailed(request=request) orig_timestamp = Timestamp(orig_metadata.get("X-Timestamp", 0)) if orig_timestamp >= req_timestamp: return HTTPConflict(request=request, headers={"X-Backend-Timestamp": orig_timestamp.internal}) orig_delete_at = int(orig_metadata.get("X-Delete-At") or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 try: with disk_file.create(size=fsize) as writer: upload_size = 0 def timeout_reader(): with ChunkReadTimeout(self.client_timeout): return request.environ["wsgi.input"].read(self.network_chunk_size) try: for chunk in iter(lambda: timeout_reader(), ""): start_time = time.time() if start_time > upload_expiration: self.logger.increment("PUT.timeouts") return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time except ChunkReadTimeout: return HTTPRequestTimeout(request=request) if upload_size: self.logger.transfer_rate("PUT." + device + ".timing", elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if "etag" in request.headers and request.headers["etag"].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { "X-Timestamp": request.timestamp.internal, "Content-Type": request.headers["content-type"], "ETag": etag, "Content-Length": str(upload_size), } metadata.update(val for val in request.headers.iteritems() if is_sys_or_user_meta("object", val[0])) headers_to_copy = request.headers.get("X-Backend-Replication-Headers", "").split() + list( self.allowed_headers ) for header_key in headers_to_copy: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) except (DiskFileXattrNotSupported, DiskFileNoSpace): return HTTPInsufficientStorage(drive=device, request=request) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update("PUT", new_delete_at, account, container, obj, request, device, policy_idx) if orig_delete_at: self.delete_at_update("DELETE", orig_delete_at, account, container, obj, request, device, policy_idx) self.container_update( "PUT", account, container, obj, request, HeaderKeyDict( { "x-size": metadata["Content-Length"], "x-content-type": metadata["Content-Type"], "x-timestamp": metadata["X-Timestamp"], "x-etag": metadata["ETag"], } ), device, policy_idx, ) return HTTPCreated(request=request, etag=etag) @public @timing_stats() def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = get_name_and_placement(request, 5, 5, True) keep_cache = self.keep_cache_private or ( "X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers ) try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata["Content-Length"]) file_x_ts = Timestamp(metadata["X-Timestamp"]) keep_cache = self.keep_cache_private or ( "X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers ) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True ) response.headers["Content-Type"] = metadata.get("Content-Type", "application/octet-stream") for key, value in metadata.iteritems(): if is_sys_or_user_meta("object", key) or key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata["ETag"] response.last_modified = math.ceil(float(file_x_ts)) response.content_length = obj_size try: response.content_encoding = metadata["Content-Encoding"] except KeyError: pass response.headers["X-Timestamp"] = file_x_ts.normal response.headers["X-Backend-Timestamp"] = file_x_ts.internal resp = request.get_response(response) except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, "timestamp"): headers["X-Backend-Timestamp"] = e.timestamp.internal resp = HTTPNotFound(request=request, headers=headers, conditional_response=True) return resp @public @timing_stats(sample_rate=0.8) def HEAD(self, request): """Handle HTTP HEAD requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = get_name_and_placement(request, 5, 5, True) try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, "timestamp"): headers["X-Backend-Timestamp"] = e.timestamp.internal return HTTPNotFound(request=request, headers=headers, conditional_response=True) response = Response(request=request, conditional_response=True) response.headers["Content-Type"] = metadata.get("Content-Type", "application/octet-stream") for key, value in metadata.iteritems(): if is_sys_or_user_meta("object", key) or key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata["ETag"] ts = Timestamp(metadata["X-Timestamp"]) response.last_modified = math.ceil(float(ts)) # Needed for container sync feature response.headers["X-Timestamp"] = ts.normal response.headers["X-Backend-Timestamp"] = ts.internal response.content_length = int(metadata["Content-Length"]) try: response.content_encoding = metadata["Content-Encoding"] except KeyError: pass return response @public @timing_stats() def DELETE(self, request): """Handle HTTP DELETE requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except DiskFileExpired as e: orig_timestamp = e.timestamp orig_metadata = e.metadata response_class = HTTPNotFound except DiskFileDeleted as e: orig_timestamp = e.timestamp orig_metadata = {} response_class = HTTPNotFound except (DiskFileNotExist, DiskFileQuarantined): orig_timestamp = 0 orig_metadata = {} response_class = HTTPNotFound else: orig_timestamp = Timestamp(orig_metadata.get("X-Timestamp", 0)) if orig_timestamp < req_timestamp: response_class = HTTPNoContent else: response_class = HTTPConflict response_timestamp = max(orig_timestamp, req_timestamp) orig_delete_at = int(orig_metadata.get("X-Delete-At") or 0) try: req_if_delete_at_val = request.headers["x-if-delete-at"] req_if_delete_at = int(req_if_delete_at_val) except KeyError: pass except ValueError: return HTTPBadRequest(request=request, body="Bad X-If-Delete-At header value") else: # request includes x-if-delete-at; we must not place a tombstone # if we can not verify the x-if-delete-at time if not orig_timestamp: # no object found at all return HTTPNotFound() if orig_delete_at != req_if_delete_at: return HTTPPreconditionFailed(request=request, body="X-If-Delete-At and X-Delete-At do not match") else: # differentiate success from no object at all response_class = HTTPNoContent if orig_delete_at: self.delete_at_update("DELETE", orig_delete_at, account, container, obj, request, device, policy_idx) if orig_timestamp < req_timestamp: disk_file.delete(req_timestamp) self.container_update( "DELETE", account, container, obj, request, HeaderKeyDict({"x-timestamp": req_timestamp.internal}), device, policy_idx, ) return response_class(request=request, headers={"X-Backend-Timestamp": response_timestamp.internal}) @public @replication @timing_stats(sample_rate=0.1) def REPLICATE(self, request): """ Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. """ device, partition, suffix, policy_idx = get_name_and_placement(request, 2, 3, True) try: hashes = self._diskfile_mgr.get_hashes(device, partition, suffix, policy_idx) except DiskFileDeviceUnavailable: resp = HTTPInsufficientStorage(drive=device, request=request) else: resp = Response(body=pickle.dumps(hashes)) return resp @public @replication @timing_stats(sample_rate=0.1) def REPLICATION(self, request): return Response(app_iter=ssync_receiver.Receiver(self, request)()) def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get("x-trans-id", None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body="Invalid UTF8 or contains NULL") else: try: # disallow methods which have not been marked 'public' try: if req.method not in self.allowed_methods: raise AttributeError("Not allowed method.") except AttributeError: res = HTTPMethodNotAllowed() else: method = getattr(self, req.method) res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception( _("ERROR __call__ error with %(method)s" " %(path)s "), {"method": req.method, "path": req.path} ) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = get_log_line(req, res, trans_time, "") if req.method in ("REPLICATE", "REPLICATION") or "X-Backend-Replication" in req.headers: self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ("PUT", "DELETE"): slow = self.slow - trans_time if slow > 0: sleep(slow) # To be able to zero-copy send the object, we need a few things. # First, we have to be responding successfully to a GET, or else we're # not sending the object. Second, we have to be able to extract the # socket file descriptor from the WSGI input object. Third, the # diskfile has to support zero-copy send. # # There's a good chance that this could work for 206 responses too, # but the common case is sending the whole object, so we'll start # there. if req.method == "GET" and res.status_int == 200 and isinstance(env["wsgi.input"], wsgi.Input): app_iter = getattr(res, "app_iter", None) checker = getattr(app_iter, "can_zero_copy_send", None) if checker and checker(): # For any kind of zero-copy thing like sendfile or splice, we # need the file descriptor. Eventlet doesn't provide a clean # way of getting that, so we resort to this. wsock = env["wsgi.input"].get_socket() wsockfd = wsock.fileno() # Don't call zero_copy_send() until after we force the HTTP # headers out of Eventlet and into the socket. def zero_copy_iter(): # If possible, set TCP_CORK so that headers don't # immediately go on the wire, but instead, wait for some # response body to make the TCP frames as large as # possible (and hence as few packets as possible). # # On non-Linux systems, we might consider TCP_NODELAY, but # since the only known zero-copy-capable diskfile uses # Linux-specific syscalls, we'll defer that work until # someone needs it. if hasattr(socket, "TCP_CORK"): wsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_CORK, 1) yield EventletPlungerString() try: app_iter.zero_copy_send(wsockfd) except Exception: self.logger.exception("zero_copy_send() blew up") raise yield "" # Get headers ready to go out res(env, start_response) return zero_copy_iter() else: return res(env, start_response) else: return res(env, start_response)
class ObjectController(object): """Implements the WSGI application for the Swift Object Server.""" def __init__(self, conf): """ Creates a new WSGI application for the Swift Object Server. An example configuration is given at <source-dir>/etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ self.logger = get_logger(conf, log_route='object-server') self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.network_chunk_size = int(conf.get('network_chunk_size', 65536)) self.log_requests = config_true_value(conf.get('log_requests', 'true')) self.max_upload_time = int(conf.get('max_upload_time', 86400)) self.slow = int(conf.get('slow', 0)) self.keep_cache_private = \ config_true_value(conf.get('keep_cache_private', 'false')) replication_server = conf.get('replication_server', None) if replication_server is not None: replication_server = config_true_value(replication_server) self.replication_server = replication_server default_allowed_headers = ''' content-disposition, content-encoding, x-delete-at, x-object-manifest, x-static-large-object, ''' extra_allowed_headers = [ header.strip().lower() for header in conf.get( 'allowed_headers', default_allowed_headers).split(',') if header.strip() ] self.allowed_headers = set() for header in extra_allowed_headers: if header not in DATAFILE_SYSTEM_META: self.allowed_headers.add(header) self.expiring_objects_account = \ (conf.get('auto_create_account_prefix') or '.') + \ 'expiring_objects' self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) # Initialization was successful, so now apply the network chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because the primary motivation for this is to optimize how data # is written back to the proxy server, we could use the value from the # disk_chunk_size parameter. However, it affects all created sockets # using this class so we have chosen to tie it to the # network_chunk_size parameter value instead. socket._fileobject.default_bufsize = self.network_chunk_size # Provide further setup sepecific to an object server implemenation. self.setup(conf) def setup(self, conf): """ Implementation specific setup. This method is called at the very end by the constructor to allow a specific implementation to modify existing attributes or add its own attributes. :param conf: WSGI configuration parameter """ # Common on-disk hierarchy shared across account, container and object # servers. self._diskfile_mgr = DiskFileManager(conf, self.logger) def get_diskfile(self, device, partition, account, container, obj, **kwargs): """ Utility method for instantiating a DiskFile object supporting a given REST API. An implementation of the object server that wants to use a different DiskFile class would simply over-ride this method to provide that behavior. """ return self._diskfile_mgr.get_diskfile(device, partition, account, container, obj, **kwargs) def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ headers_out['user-agent'] = 'obj-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error( _('ERROR Container update failed ' '(saving for async update later): %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), { 'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice }) except (Exception, Timeout): self.logger.exception( _('ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s (saving for async update later)' ), { 'ip': ip, 'port': port, 'dev': contdevice }) data = { 'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out } timestamp = headers_out['x-timestamp'] self._diskfile_mgr.pickle_async_update(objdevice, account, container, obj, data, timestamp) def container_update(self, op, account, container, obj, request, headers_out, objdevice): """ Update the container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request object driving the update :param headers_out: dictionary of headers to send in the container request(s) :param objdevice: device name that the object is in """ headers_in = request.headers conthosts = [ h.strip() for h in headers_in.get('X-Container-Host', '').split(',') ] contdevices = [ d.strip() for d in headers_in.get('X-Container-Device', '').split(',') ] contpartition = headers_in.get('X-Container-Partition', '') if len(conthosts) != len(contdevices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. self.logger.error( _('ERROR Container update failed: different ' 'numbers of hosts and devices in request: ' '"%s" vs "%s"') % (headers_in.get('X-Container-Host', ''), headers_in.get('X-Container-Device', ''))) return if contpartition: updates = zip(conthosts, contdevices) else: updates = [] headers_out['x-trans-id'] = headers_in.get('x-trans-id', '-') headers_out['referer'] = request.as_referer() for conthost, contdevice in updates: self.async_update(op, account, container, obj, conthost, contpartition, contdevice, headers_out, objdevice) def delete_at_update(self, op, delete_at, account, container, obj, request, objdevice): """ Update the expiring objects container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param delete_at: scheduled delete in UNIX seconds, int :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request driving the update :param objdevice: device name that the object is in """ # Quick cap that will work from now until Sat Nov 20 17:46:39 2286 # At that time, Swift will be so popular and pervasive I will have # created income for thousands of future programmers. delete_at = max(min(delete_at, 9999999999), 0) updates = [(None, None)] partition = None hosts = contdevices = [None] headers_in = request.headers headers_out = HeaderKeyDict({ 'x-timestamp': headers_in['x-timestamp'], 'x-trans-id': headers_in.get('x-trans-id', '-'), 'referer': request.as_referer() }) if op != 'DELETE': delete_at_container = headers_in.get('X-Delete-At-Container', None) if not delete_at_container: self.logger.warning( 'X-Delete-At-Container header must be specified for ' 'expiring objects background %s to work properly. Making ' 'best guess as to the container name for now.' % op) # TODO(gholt): In a future release, change the above warning to # a raised exception and remove the guess code below. delete_at_container = str( delete_at / self.expiring_objects_container_divisor * self.expiring_objects_container_divisor) partition = headers_in.get('X-Delete-At-Partition', None) hosts = headers_in.get('X-Delete-At-Host', '') contdevices = headers_in.get('X-Delete-At-Device', '') updates = [ upd for upd in zip((h.strip() for h in hosts.split(',')), ( c.strip() for c in contdevices.split(','))) if all(upd) and partition ] if not updates: updates = [(None, None)] headers_out['x-size'] = '0' headers_out['x-content-type'] = 'text/plain' headers_out['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e' else: # DELETEs of old expiration data have no way of knowing what the # old X-Delete-At-Container was at the time of the initial setting # of the data, so a best guess is made here. # Worst case is a DELETE is issued now for something that doesn't # exist there and the original data is left where it is, where # it will be ignored when the expirer eventually tries to issue the # object DELETE later since the X-Delete-At value won't match up. delete_at_container = str(delete_at / self.expiring_objects_container_divisor * self.expiring_objects_container_divisor) for host, contdevice in updates: self.async_update( op, self.expiring_objects_account, delete_at_container, '%s-%s/%s/%s' % (delete_at, account, container, obj), host, partition, contdevice, headers_out, objdevice) @public @timing_stats() def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = orig_metadata.get('X-Timestamp', '0') if orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) metadata = {'X-Timestamp': request.headers['x-timestamp']} metadata.update(val for val in request.headers.iteritems() if val[0].startswith('X-Object-Meta-')) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device) try: disk_file.write_metadata(metadata) except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) else: return HTTPAccepted(request=request) @public @timing_stats() def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') error_response = check_object_creation(request, obj) if error_response: return error_response new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} orig_timestamp = orig_metadata.get('X-Timestamp') if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 try: with disk_file.create(size=fsize) as writer: upload_size = 0 reader = request.environ['wsgi.input'].read for chunk in iter(lambda: reader(self.network_chunk_size), ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) sleep() elapsed_time += time.time() - start_time if upload_size: self.logger.transfer_rate('PUT.' + device + '.timing', elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { 'X-Timestamp': request.headers['x-timestamp'], 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.iteritems() if val[0].lower().startswith('x-object-meta-') and len(val[0]) > 14) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) except DiskFileNoSpace: return HTTPInsufficientStorage(drive=device, request=request) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device) if not orig_timestamp or \ orig_timestamp < request.headers['x-timestamp']: self.container_update( 'PUT', account, container, obj, request, HeaderKeyDict({ 'x-size': metadata['Content-Length'], 'x-content-type': metadata['Content-Type'], 'x-timestamp': metadata['X-Timestamp'], 'x-etag': metadata['ETag'] }), device) return HTTPCreated(request=request, etag=etag) @public @timing_stats() def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) if request.headers.get('if-match') not in (None, '*') and \ metadata['ETag'] not in request.if_match: return HTTPPreconditionFailed(request=request) if request.headers.get('if-none-match') is not None: if metadata['ETag'] in request.if_none_match: resp = HTTPNotModified(request=request) resp.etag = metadata['ETag'] return resp file_x_ts = metadata['X-Timestamp'] file_x_ts_flt = float(file_x_ts) try: if_unmodified_since = request.if_unmodified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC) if if_unmodified_since and file_x_ts_utc > if_unmodified_since: return HTTPPreconditionFailed(request=request) try: if_modified_since = request.if_modified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if if_modified_since and file_x_ts_utc < if_modified_since: return HTTPNotModified(request=request) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) response = Response(app_iter=disk_file.reader( iter_hook=sleep, keep_cache=keep_cache), request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if key.lower().startswith('x-object-meta-') or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = file_x_ts_flt response.content_length = obj_size try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts resp = request.get_response(response) except DiskFileNotExist: if request.headers.get('if-match') == '*': resp = HTTPPreconditionFailed(request=request) else: resp = HTTPNotFound(request=request) except DiskFileQuarantined: resp = HTTPNotFound(request=request) return resp @public @timing_stats(sample_rate=0.8) def HEAD(self, request): """Handle HTTP HEAD requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) response = Response(request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if key.lower().startswith('x-object-meta-') or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] ts = metadata['X-Timestamp'] response.last_modified = float(ts) # Needed for container sync feature response.headers['X-Timestamp'] = ts response.content_length = int(metadata['Content-Length']) try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass return response @public @timing_stats() def DELETE(self, request): """Handle HTTP DELETE requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileDeleted as e: orig_timestamp = e.timestamp orig_metadata = {} response_class = HTTPNotFound except (DiskFileNotExist, DiskFileQuarantined): orig_timestamp = 0 orig_metadata = {} response_class = HTTPNotFound else: orig_timestamp = orig_metadata.get('X-Timestamp', 0) if orig_timestamp < request.headers['x-timestamp']: response_class = HTTPNoContent else: response_class = HTTPConflict if 'x-if-delete-at' in request.headers and \ int(request.headers['x-if-delete-at']) != \ int(orig_metadata.get('X-Delete-At') or 0): return HTTPPreconditionFailed( request=request, body='X-If-Delete-At and X-Delete-At do not match') orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device) req_timestamp = request.headers['X-Timestamp'] if orig_timestamp < req_timestamp: disk_file.delete(req_timestamp) self.container_update( 'DELETE', account, container, obj, request, HeaderKeyDict({'x-timestamp': req_timestamp}), device) return response_class(request=request) @public @replication @timing_stats(sample_rate=0.1) def REPLICATE(self, request): """ Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. """ device, partition, suffix = split_and_validate_path( request, 2, 3, True) try: hashes = self._diskfile_mgr.get_hashes(device, partition, suffix) except DiskFileDeviceUnavailable: resp = HTTPInsufficientStorage(drive=device, request=request) else: resp = Response(body=pickle.dumps(hashes)) return resp def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception( _('ERROR __call__ error with %(method)s' ' %(path)s '), { 'method': req.method, 'path': req.path }) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f' % ( req.remote_addr, time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.referer or '-', req.headers.get( 'x-trans-id', '-'), req.user_agent or '-', trans_time) if req.method == 'REPLICATE': self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time if slow > 0: sleep(slow) return res(env, start_response)
class ObjectController(BaseStorageServer): """Implements the WSGI application for the Swift Object Server.""" server_type = 'object-server' def __init__(self, conf, logger=None): """ Creates a new WSGI application for the Swift Object Server. An example configuration is given at <source-dir>/etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ super(ObjectController, self).__init__(conf) self.logger = logger or get_logger(conf, log_route='object-server') self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) self.network_chunk_size = int(conf.get('network_chunk_size', 65536)) self.log_requests = config_true_value(conf.get('log_requests', 'true')) self.max_upload_time = int(conf.get('max_upload_time', 86400)) self.slow = int(conf.get('slow', 0)) self.keep_cache_private = \ config_true_value(conf.get('keep_cache_private', 'false')) default_allowed_headers = ''' content-disposition, content-encoding, x-delete-at, x-object-manifest, x-static-large-object, ''' extra_allowed_headers = [ header.strip().lower() for header in conf.get( 'allowed_headers', default_allowed_headers).split(',') if header.strip() ] self.allowed_headers = set() for header in extra_allowed_headers: if header not in DATAFILE_SYSTEM_META: self.allowed_headers.add(header) self.auto_create_account_prefix = \ conf.get('auto_create_account_prefix') or '.' self.expiring_objects_account = self.auto_create_account_prefix + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) # Initialization was successful, so now apply the network chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because the primary motivation for this is to optimize how data # is written back to the proxy server, we could use the value from the # disk_chunk_size parameter. However, it affects all created sockets # using this class so we have chosen to tie it to the # network_chunk_size parameter value instead. socket._fileobject.default_bufsize = self.network_chunk_size # Provide further setup specific to an object server implementation. self.setup(conf) def setup(self, conf): """ Implementation specific setup. This method is called at the very end by the constructor to allow a specific implementation to modify existing attributes or add its own attributes. :param conf: WSGI configuration parameter """ # Common on-disk hierarchy shared across account, container and object # servers. self._diskfile_mgr = DiskFileManager(conf, self.logger) # This is populated by global_conf_callback way below as the semaphore # is shared by all workers. if 'replication_semaphore' in conf: # The value was put in a list so it could get past paste self.replication_semaphore = conf['replication_semaphore'][0] else: self.replication_semaphore = None self.replication_failure_threshold = int( conf.get('replication_failure_threshold') or 100) self.replication_failure_ratio = float( conf.get('replication_failure_ratio') or 1.0) def get_diskfile(self, device, partition, account, container, obj, policy_idx, **kwargs): """ Utility method for instantiating a DiskFile object supporting a given REST API. An implementation of the object server that wants to use a different DiskFile class would simply over-ride this method to provide that behavior. """ return self._diskfile_mgr.get_diskfile( device, partition, account, container, obj, policy_idx, **kwargs) def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice, policy_index): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in :param policy_index: the associated storage policy index """ headers_out['user-agent'] = 'object-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error(_( 'ERROR Container update failed ' '(saving for async update later): %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), {'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice}) except (Exception, Timeout): self.logger.exception(_( 'ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s (saving for async update later)'), {'ip': ip, 'port': port, 'dev': contdevice}) data = {'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out} timestamp = headers_out['x-timestamp'] self._diskfile_mgr.pickle_async_update(objdevice, account, container, obj, data, timestamp, policy_index) def container_update(self, op, account, container, obj, request, headers_out, objdevice, policy_idx): """ Update the container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request object driving the update :param headers_out: dictionary of headers to send in the container request(s) :param objdevice: device name that the object is in """ headers_in = request.headers conthosts = [h.strip() for h in headers_in.get('X-Container-Host', '').split(',')] contdevices = [d.strip() for d in headers_in.get('X-Container-Device', '').split(',')] contpartition = headers_in.get('X-Container-Partition', '') if len(conthosts) != len(contdevices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. self.logger.error(_('ERROR Container update failed: different ' 'numbers of hosts and devices in request: ' '"%s" vs "%s"') % (headers_in.get('X-Container-Host', ''), headers_in.get('X-Container-Device', ''))) return if contpartition: updates = zip(conthosts, contdevices) else: updates = [] headers_out['x-trans-id'] = headers_in.get('x-trans-id', '-') headers_out['referer'] = request.as_referer() headers_out['X-Backend-Storage-Policy-Index'] = policy_idx for conthost, contdevice in updates: self.async_update(op, account, container, obj, conthost, contpartition, contdevice, headers_out, objdevice, policy_idx) def delete_at_update(self, op, delete_at, account, container, obj, request, objdevice, policy_index): """ Update the expiring objects container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param delete_at: scheduled delete in UNIX seconds, int :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request driving the update :param objdevice: device name that the object is in :param policy_index: the policy index to be used for tmp dir """ if config_true_value( request.headers.get('x-backend-replication', 'f')): return delete_at = normalize_delete_at_timestamp(delete_at) updates = [(None, None)] partition = None hosts = contdevices = [None] headers_in = request.headers headers_out = HeaderKeyDict({ # system accounts are always Policy-0 'X-Backend-Storage-Policy-Index': 0, 'x-timestamp': request.timestamp.internal, 'x-trans-id': headers_in.get('x-trans-id', '-'), 'referer': request.as_referer()}) if op != 'DELETE': delete_at_container = headers_in.get('X-Delete-At-Container', None) if not delete_at_container: self.logger.warning( 'X-Delete-At-Container header must be specified for ' 'expiring objects background %s to work properly. Making ' 'best guess as to the container name for now.' % op) # TODO(gholt): In a future release, change the above warning to # a raised exception and remove the guess code below. delete_at_container = get_expirer_container( delete_at, self.expiring_objects_container_divisor, account, container, obj) partition = headers_in.get('X-Delete-At-Partition', None) hosts = headers_in.get('X-Delete-At-Host', '') contdevices = headers_in.get('X-Delete-At-Device', '') updates = [upd for upd in zip((h.strip() for h in hosts.split(',')), (c.strip() for c in contdevices.split(','))) if all(upd) and partition] if not updates: updates = [(None, None)] headers_out['x-size'] = '0' headers_out['x-content-type'] = 'text/plain' headers_out['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e' else: # DELETEs of old expiration data have no way of knowing what the # old X-Delete-At-Container was at the time of the initial setting # of the data, so a best guess is made here. # Worst case is a DELETE is issued now for something that doesn't # exist there and the original data is left where it is, where # it will be ignored when the expirer eventually tries to issue the # object DELETE later since the X-Delete-At value won't match up. delete_at_container = get_expirer_container( delete_at, self.expiring_objects_container_divisor, account, container, obj) delete_at_container = normalize_delete_at_timestamp( delete_at_container) for host, contdevice in updates: self.async_update( op, self.expiring_objects_account, delete_at_container, '%s-%s/%s/%s' % (delete_at, account, container, obj), host, partition, contdevice, headers_out, objdevice, policy_index) @public @timing_stats() def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: return HTTPConflict( request=request, headers={'X-Backend-Timestamp': orig_timestamp.internal}) metadata = {'X-Timestamp': req_timestamp.internal} metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device, policy_idx) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy_idx) try: disk_file.write_metadata(metadata) except (DiskFileXattrNotSupported, DiskFileNoSpace): return HTTPInsufficientStorage(drive=device, request=request) return HTTPAccepted(request=request) @public @timing_stats() def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" a=time.time() #with open("/home/ubuntu/WSTORAGE.txt", "a") as tran_file: # tran_file.write("A = "+str(request.path)+"="+str(datetime.now())+"***") #with open("/home/ubuntu/spawn.txt", "a") as tran_file: # tran_file.write("At PUT Datetime ="+str(datetime.now())+" \n") device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) error_response = check_object_creation(request, obj) #with open("/home/ubuntu/WSTORAGE.txt", "a") as tran_file: # tran_file.write("B = "+str(request.path)+"="+str(datetime.now())+"***") if error_response: return error_response new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') #with open("/home/ubuntu/WSTORAGE.txt", "a") as tran_file: # tran_file.write("C = "+str(request.path)+"="+str(datetime.now())+"***") try: disk_file = self.get_diskfile( device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) #with open("/home/ubuntu/WSTORAGE.txt", "a") as tran_file: # tran_file.write("D = "+str(request.path)+"="+str(datetime.now())+"***") try: orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} #with open("/home/ubuntu/WSTORAGE.txt", "a") as tran_file: # tran_file.write("E = "+str(request.path)+"="+str(datetime.now())+"***") # Checks for If-None-Match if request.if_none_match is not None and orig_metadata: if '*' in request.if_none_match: # File exists already so return 412 return HTTPPreconditionFailed(request=request) if orig_metadata.get('ETag') in request.if_none_match: # The current ETag matches, so return 412 return HTTPPreconditionFailed(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: return HTTPConflict( request=request, headers={'X-Backend-Timestamp': orig_timestamp.internal}) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 #with open("/home/ubuntu/WSTORAGE.txt", "a") as tran_file: # tran_file.write("F = "+str(request.path)+"="+str(datetime.now())+"***") try: with disk_file.create(size=fsize) as writer: upload_size = 0 def timeout_reader(): with ChunkReadTimeout(self.client_timeout): return request.environ['wsgi.input'].read( self.network_chunk_size) try: for chunk in iter(lambda: timeout_reader(), ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time except ChunkReadTimeout: return HTTPRequestTimeout(request=request) #with open("/home/ubuntu/WSTORAGE.txt", "a") as tran_file: # tran_file.write("G = "+str(request.path)+"="+str(datetime.now())+"***") if upload_size: self.logger.transfer_rate( 'PUT.' + device + '.timing', elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { 'X-Timestamp': request.timestamp.internal, 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.iteritems() if is_sys_or_user_meta('object', val[0])) headers_to_copy = ( request.headers.get( 'X-Backend-Replication-Headers', '').split() + list(self.allowed_headers)) for header_key in headers_to_copy: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] #with open("/home/ubuntu/WSTORAGE.txt", "a") as tran_file: # tran_file.write("H = "+str(request.path)+"="+str(datetime.now())+"***") writer.put(metadata) #with open("/home/ubuntu/WSTORAGE.txt", "a") as tran_file: # tran_file.write("I = "+str(request.path)+"="+str(datetime.now())+"***") except (DiskFileXattrNotSupported, DiskFileNoSpace): return HTTPInsufficientStorage(drive=device, request=request) #with open("/home/ubuntu/WSTORAGE.txt", "a") as tran_file: # tran_file.write("J = "+str(request.path)+"="+str(datetime.now())+"***") if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update( 'PUT', new_delete_at, account, container, obj, request, device, policy_idx) if orig_delete_at: self.delete_at_update( 'DELETE', orig_delete_at, account, container, obj, request, device, policy_idx) #with open("/home/ubuntu/WSTORAGE.txt", "a") as tran_file: # tran_file.write("K = "+str(request.path)+"="+str(datetime.now())+"***") self.container_update( 'PUT', account, container, obj, request, HeaderKeyDict({ 'x-size': metadata['Content-Length'], 'x-content-type': metadata['Content-Type'], 'x-timestamp': metadata['X-Timestamp'], 'x-etag': metadata['ETag']}), device, policy_idx) #with open("/home/ubuntu/WSTORAGE.txt", "a") as tran_file: # tran_file.write("L = "+str(request.path)+"="+str(datetime.now())+"***") #with open("/home/ubuntu/obj_create.txt", "a") as tran_file: # tran_file.write("Started = "+str(datetime.now())+"\n") #a=2 #num=22227727 #for a in range(a, num): # if a % num == 0: # print('not prime') # break #else: # loop not exited via break # print('prime') #time.sleep(0.1) #with open("/home/ubuntu/obj_create.txt", "a") as tran_file: # tran_file.write("Stopped = "+str(datetime.now())+"\n") b=time.time()-a #with open("/home/ubuntu/WSTORAGE.txt", "a") as tran_file: # tran_file.write("Total PUT duration of obj = "+str(request.path)+"="+str(b)+"="+str(datetime.now())+"\n") return HTTPCreated(request=request, etag=etag) @public @timing_stats() def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" #with open("/home/ubuntu/spawn-read.txt", "a") as tran_file: # tran_file.write("At Read Start Datetime = "+str(datetime.now())+"\n") a=time.time() device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile( device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) file_x_ts = Timestamp(metadata['X-Timestamp']) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_sys_or_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(float(file_x_ts)) response.content_length = obj_size try: response.content_encoding = metadata[ 'Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts.normal response.headers['X-Backend-Timestamp'] = file_x_ts.internal resp = request.get_response(response) except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): headers['X-Backend-Timestamp'] = e.timestamp.internal resp = HTTPNotFound(request=request, headers=headers, conditional_response=True) #with open("/home/ubuntu/spawn-read.txt", "a") as tran_file: # tran_file.write("At Read Delay start Datetime = "+str(datetime.now())+"\n") #a=2 #num=22227727 #for a in range(a, num): # if a % num == 0: # print('not prime') # break # else: # loop not exited via break # print('prime') #time.sleep(2) #with open("/home/ubuntu/spawn-read.txt", "a") as tran_file: # tran_file.write("At Read End Datetime = "+str(datetime.now())+"\n") b=time.time()-a #with open("/home/ubuntu/RSTORAGE.txt", "a") as tran_file: # tran_file.write("Total GET duration of obj = "+str(request.path)+"="+str(b)+"="+str(datetime.now())+"\n") return resp @public @timing_stats(sample_rate=0.8) def HEAD(self, request): """Handle HTTP HEAD requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) try: disk_file = self.get_diskfile( device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): headers['X-Backend-Timestamp'] = e.timestamp.internal return HTTPNotFound(request=request, headers=headers, conditional_response=True) response = Response(request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_sys_or_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] ts = Timestamp(metadata['X-Timestamp']) response.last_modified = math.ceil(float(ts)) # Needed for container sync feature response.headers['X-Timestamp'] = ts.normal response.headers['X-Backend-Timestamp'] = ts.internal response.content_length = int(metadata['Content-Length']) try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass return response @public @timing_stats() def DELETE(self, request): """Handle HTTP DELETE requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) try: disk_file = self.get_diskfile( device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except DiskFileExpired as e: orig_timestamp = e.timestamp orig_metadata = e.metadata response_class = HTTPNotFound except DiskFileDeleted as e: orig_timestamp = e.timestamp orig_metadata = {} response_class = HTTPNotFound except (DiskFileNotExist, DiskFileQuarantined): orig_timestamp = 0 orig_metadata = {} response_class = HTTPNotFound else: orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp < req_timestamp: response_class = HTTPNoContent else: response_class = HTTPConflict response_timestamp = max(orig_timestamp, req_timestamp) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) try: req_if_delete_at_val = request.headers['x-if-delete-at'] req_if_delete_at = int(req_if_delete_at_val) except KeyError: pass except ValueError: return HTTPBadRequest( request=request, body='Bad X-If-Delete-At header value') else: # request includes x-if-delete-at; we must not place a tombstone # if we can not verify the x-if-delete-at time if not orig_timestamp: # no object found at all return HTTPNotFound() if orig_delete_at != req_if_delete_at: return HTTPPreconditionFailed( request=request, body='X-If-Delete-At and X-Delete-At do not match') else: # differentiate success from no object at all response_class = HTTPNoContent if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy_idx) if orig_timestamp < req_timestamp: disk_file.delete(req_timestamp) self.container_update( 'DELETE', account, container, obj, request, HeaderKeyDict({'x-timestamp': req_timestamp.internal}), device, policy_idx) return response_class( request=request, headers={'X-Backend-Timestamp': response_timestamp.internal}) @public @replication @timing_stats(sample_rate=0.1) def REPLICATE(self, request): """ Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. """ device, partition, suffix, policy_idx = \ get_name_and_placement(request, 2, 3, True) try: hashes = self._diskfile_mgr.get_hashes(device, partition, suffix, policy_idx) except DiskFileDeviceUnavailable: resp = HTTPInsufficientStorage(drive=device, request=request) else: resp = Response(body=pickle.dumps(hashes)) return resp @public @replication @timing_stats(sample_rate=0.1) def REPLICATION(self, request): return Response(app_iter=ssync_receiver.Receiver(self, request)()) def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" with open("/home/ubuntu/spawn.txt", "a") as tran_file: tran_file.write("At Start Datetime = "+str(datetime.now())+"\n") start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: if req.method not in self.allowed_methods: raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: method = getattr(self, req.method) res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception(_( 'ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = get_log_line(req, res, trans_time, '') if req.method in ('REPLICATE', 'REPLICATION') or \ 'X-Backend-Replication' in req.headers: self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time with open("/home/ubuntu/spawn.txt", "a") as tran_file: tran_file.write("Before slow check Datetime = "+str(datetime.now())+" Slow="+str(slow)+"\n") if slow > 0: sleep(slow) # To be able to zero-copy send the object, we need a few things. # First, we have to be responding successfully to a GET, or else we're # not sending the object. Second, we have to be able to extract the # socket file descriptor from the WSGI input object. Third, the # diskfile has to support zero-copy send. # # There's a good chance that this could work for 206 responses too, # but the common case is sending the whole object, so we'll start # there. if req.method == 'GET' and res.status_int == 200 and \ isinstance(env['wsgi.input'], wsgi.Input): app_iter = getattr(res, 'app_iter', None) checker = getattr(app_iter, 'can_zero_copy_send', None) if checker and checker(): # For any kind of zero-copy thing like sendfile or splice, we # need the file descriptor. Eventlet doesn't provide a clean # way of getting that, so we resort to this. wsock = env['wsgi.input'].get_socket() wsockfd = wsock.fileno() # Don't call zero_copy_send() until after we force the HTTP # headers out of Eventlet and into the socket. def zero_copy_iter(): # If possible, set TCP_CORK so that headers don't # immediately go on the wire, but instead, wait for some # response body to make the TCP frames as large as # possible (and hence as few packets as possible). # # On non-Linux systems, we might consider TCP_NODELAY, but # since the only known zero-copy-capable diskfile uses # Linux-specific syscalls, we'll defer that work until # someone needs it. if hasattr(socket, 'TCP_CORK'): wsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_CORK, 1) yield EventletPlungerString() try: app_iter.zero_copy_send(wsockfd) except Exception: self.logger.exception("zero_copy_send() blew up") raise yield '' # Get headers ready to go out res(env, start_response) return zero_copy_iter() else: return res(env, start_response) else: return res(env, start_response)
class ObjectController(object): """Implements the WSGI application for the Swift Object Server.""" def __init__(self, conf, logger=None): """ Creates a new WSGI application for the Swift Object Server. An example configuration is given at <source-dir>/etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ self.logger = logger or get_logger(conf, log_route="object-server") self.node_timeout = int(conf.get("node_timeout", 3)) self.conn_timeout = float(conf.get("conn_timeout", 0.5)) self.client_timeout = int(conf.get("client_timeout", 60)) self.disk_chunk_size = int(conf.get("disk_chunk_size", 65536)) self.network_chunk_size = int(conf.get("network_chunk_size", 65536)) self.log_requests = config_true_value(conf.get("log_requests", "true")) self.max_upload_time = int(conf.get("max_upload_time", 86400)) self.slow = int(conf.get("slow", 0)) self.keep_cache_private = config_true_value(conf.get("keep_cache_private", "false")) replication_server = conf.get("replication_server", None) if replication_server is not None: replication_server = config_true_value(replication_server) self.replication_server = replication_server default_allowed_headers = """ content-disposition, content-encoding, x-delete-at, x-object-manifest, x-static-large-object, """ extra_allowed_headers = [ header.strip().lower() for header in conf.get("allowed_headers", default_allowed_headers).split(",") if header.strip() ] self.allowed_headers = set() for header in extra_allowed_headers: if header not in DATAFILE_SYSTEM_META: self.allowed_headers.add(header) self.expiring_objects_account = (conf.get("auto_create_account_prefix") or ".") + "expiring_objects" self.expiring_objects_container_divisor = int(conf.get("expiring_objects_container_divisor") or 86400) # Initialization was successful, so now apply the network chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because the primary motivation for this is to optimize how data # is written back to the proxy server, we could use the value from the # disk_chunk_size parameter. However, it affects all created sockets # using this class so we have chosen to tie it to the # network_chunk_size parameter value instead. socket._fileobject.default_bufsize = self.network_chunk_size # Provide further setup sepecific to an object server implemenation. self.setup(conf) def setup(self, conf): """ Implementation specific setup. This method is called at the very end by the constructor to allow a specific implementation to modify existing attributes or add its own attributes. :param conf: WSGI configuration parameter """ # Common on-disk hierarchy shared across account, container and object # servers. self._diskfile_mgr = DiskFileManager(conf, self.logger) # This is populated by global_conf_callback way below as the semaphore # is shared by all workers. if "replication_semaphore" in conf: # The value was put in a list so it could get past paste self.replication_semaphore = conf["replication_semaphore"][0] else: self.replication_semaphore = None self.replication_failure_threshold = int(conf.get("replication_failure_threshold") or 100) self.replication_failure_ratio = float(conf.get("replication_failure_ratio") or 1.0) def get_diskfile(self, device, partition, account, container, obj, **kwargs): """ Utility method for instantiating a DiskFile object supporting a given REST API. An implementation of the object server that wants to use a different DiskFile class would simply over-ride this method to provide that behavior. """ return self._diskfile_mgr.get_diskfile(device, partition, account, container, obj, **kwargs) def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ headers_out["user-agent"] = "obj-server %s" % os.getpid() full_path = "/%s/%s/%s" % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(":", 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error( _( "ERROR Container update failed " "(saving for async update later): %(status)d " "response from %(ip)s:%(port)s/%(dev)s" ), {"status": response.status, "ip": ip, "port": port, "dev": contdevice}, ) except (Exception, Timeout): self.logger.exception( _("ERROR container update failed with " "%(ip)s:%(port)s/%(dev)s (saving for async update later)"), {"ip": ip, "port": port, "dev": contdevice}, ) data = {"op": op, "account": account, "container": container, "obj": obj, "headers": headers_out} timestamp = headers_out["x-timestamp"] self._diskfile_mgr.pickle_async_update(objdevice, account, container, obj, data, timestamp) def container_update(self, op, account, container, obj, request, headers_out, objdevice): """ Update the container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request object driving the update :param headers_out: dictionary of headers to send in the container request(s) :param objdevice: device name that the object is in """ headers_in = request.headers conthosts = [h.strip() for h in headers_in.get("X-Container-Host", "").split(",")] contdevices = [d.strip() for d in headers_in.get("X-Container-Device", "").split(",")] contpartition = headers_in.get("X-Container-Partition", "") if len(conthosts) != len(contdevices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. self.logger.error( _( "ERROR Container update failed: different " "numbers of hosts and devices in request: " '"%s" vs "%s"' ) % (headers_in.get("X-Container-Host", ""), headers_in.get("X-Container-Device", "")) ) return if contpartition: updates = zip(conthosts, contdevices) else: updates = [] headers_out["x-trans-id"] = headers_in.get("x-trans-id", "-") headers_out["referer"] = request.as_referer() for conthost, contdevice in updates: self.async_update(op, account, container, obj, conthost, contpartition, contdevice, headers_out, objdevice) def delete_at_update(self, op, delete_at, account, container, obj, request, objdevice): """ Update the expiring objects container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param delete_at: scheduled delete in UNIX seconds, int :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request driving the update :param objdevice: device name that the object is in """ if config_true_value(request.headers.get("x-backend-replication", "f")): return delete_at = normalize_delete_at_timestamp(delete_at) updates = [(None, None)] partition = None hosts = contdevices = [None] headers_in = request.headers headers_out = HeaderKeyDict( { "x-timestamp": headers_in["x-timestamp"], "x-trans-id": headers_in.get("x-trans-id", "-"), "referer": request.as_referer(), } ) if op != "DELETE": delete_at_container = headers_in.get("X-Delete-At-Container", None) if not delete_at_container: self.logger.warning( "X-Delete-At-Container header must be specified for " "expiring objects background %s to work properly. Making " "best guess as to the container name for now." % op ) # TODO(gholt): In a future release, change the above warning to # a raised exception and remove the guess code below. delete_at_container = ( int(delete_at) / self.expiring_objects_container_divisor * self.expiring_objects_container_divisor ) partition = headers_in.get("X-Delete-At-Partition", None) hosts = headers_in.get("X-Delete-At-Host", "") contdevices = headers_in.get("X-Delete-At-Device", "") updates = [ upd for upd in zip((h.strip() for h in hosts.split(",")), (c.strip() for c in contdevices.split(","))) if all(upd) and partition ] if not updates: updates = [(None, None)] headers_out["x-size"] = "0" headers_out["x-content-type"] = "text/plain" headers_out["x-etag"] = "d41d8cd98f00b204e9800998ecf8427e" else: # DELETEs of old expiration data have no way of knowing what the # old X-Delete-At-Container was at the time of the initial setting # of the data, so a best guess is made here. # Worst case is a DELETE is issued now for something that doesn't # exist there and the original data is left where it is, where # it will be ignored when the expirer eventually tries to issue the # object DELETE later since the X-Delete-At value won't match up. delete_at_container = str( int(delete_at) / self.expiring_objects_container_divisor * self.expiring_objects_container_divisor ) delete_at_container = normalize_delete_at_timestamp(delete_at_container) for host, contdevice in updates: self.async_update( op, self.expiring_objects_account, delete_at_container, "%s-%s/%s/%s" % (delete_at, account, container, obj), host, partition, contdevice, headers_out, objdevice, ) @public @timing_stats() def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj = split_and_validate_path(request, 5, 5, True) if "x-timestamp" not in request.headers or not check_float(request.headers["x-timestamp"]): return HTTPBadRequest(body="Missing timestamp", request=request, content_type="text/plain") new_delete_at = int(request.headers.get("X-Delete-At") or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body="X-Delete-At in past", request=request, content_type="text/plain") try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = orig_metadata.get("X-Timestamp", "0") if orig_timestamp >= request.headers["x-timestamp"]: return HTTPConflict(request=request) metadata = {"X-Timestamp": request.headers["x-timestamp"]} metadata.update(val for val in request.headers.iteritems() if is_user_meta("object", val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get("X-Delete-At") or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update("PUT", new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update("DELETE", orig_delete_at, account, container, obj, request, device) disk_file.write_metadata(metadata) return HTTPAccepted(request=request) @public @timing_stats() def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj = split_and_validate_path(request, 5, 5, True) if "x-timestamp" not in request.headers or not check_float(request.headers["x-timestamp"]): return HTTPBadRequest(body="Missing timestamp", request=request, content_type="text/plain") error_response = check_object_creation(request, obj) if error_response: return error_response new_delete_at = int(request.headers.get("X-Delete-At") or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body="X-Delete-At in past", request=request, content_type="text/plain") try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type="text/plain") try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} orig_timestamp = orig_metadata.get("X-Timestamp") if orig_timestamp and orig_timestamp >= request.headers["x-timestamp"]: return HTTPConflict(request=request) orig_delete_at = int(orig_metadata.get("X-Delete-At") or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 try: with disk_file.create(size=fsize) as writer: upload_size = 0 reader = request.environ["wsgi.input"].read for chunk in iter(lambda: reader(self.network_chunk_size), ""): start_time = time.time() if start_time > upload_expiration: self.logger.increment("PUT.timeouts") return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time if upload_size: self.logger.transfer_rate("PUT." + device + ".timing", elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if "etag" in request.headers and request.headers["etag"].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { "X-Timestamp": request.headers["x-timestamp"], "Content-Type": request.headers["content-type"], "ETag": etag, "Content-Length": str(upload_size), } metadata.update(val for val in request.headers.iteritems() if is_user_meta("object", val[0])) for header_key in request.headers.get("X-Backend-Replication-Headers") or self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) except DiskFileNoSpace: return HTTPInsufficientStorage(drive=device, request=request) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update("PUT", new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update("DELETE", orig_delete_at, account, container, obj, request, device) self.container_update( "PUT", account, container, obj, request, HeaderKeyDict( { "x-size": metadata["Content-Length"], "x-content-type": metadata["Content-Type"], "x-timestamp": metadata["X-Timestamp"], "x-etag": metadata["ETag"], } ), device, ) return HTTPCreated(request=request, etag=etag) @public @timing_stats() def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj = split_and_validate_path(request, 5, 5, True) keep_cache = self.keep_cache_private or ( "X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers ) try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata["Content-Length"]) if request.headers.get("if-match") not in (None, "*") and metadata["ETag"] not in request.if_match: return HTTPPreconditionFailed(request=request) if request.headers.get("if-none-match") is not None: if metadata["ETag"] in request.if_none_match: resp = HTTPNotModified(request=request) resp.etag = metadata["ETag"] return resp file_x_ts = metadata["X-Timestamp"] file_x_ts_flt = float(file_x_ts) try: if_unmodified_since = request.if_unmodified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC) if if_unmodified_since and file_x_ts_utc > if_unmodified_since: return HTTPPreconditionFailed(request=request) try: if_modified_since = request.if_modified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if if_modified_since and file_x_ts_utc <= if_modified_since: return HTTPNotModified(request=request) keep_cache = self.keep_cache_private or ( "X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers ) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True ) response.headers["Content-Type"] = metadata.get("Content-Type", "application/octet-stream") for key, value in metadata.iteritems(): if is_user_meta("object", key) or key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata["ETag"] response.last_modified = math.ceil(file_x_ts_flt) response.content_length = obj_size try: response.content_encoding = metadata["Content-Encoding"] except KeyError: pass response.headers["X-Timestamp"] = file_x_ts resp = request.get_response(response) except DiskFileNotExist: if request.headers.get("if-match") == "*": resp = HTTPPreconditionFailed(request=request) else: resp = HTTPNotFound(request=request) except DiskFileQuarantined: resp = HTTPNotFound(request=request) return resp @public @timing_stats(sample_rate=0.8) def HEAD(self, request): """Handle HTTP HEAD requests for the Swift Object Server.""" device, partition, account, container, obj = split_and_validate_path(request, 5, 5, True) try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) response = Response(request=request, conditional_response=True) response.headers["Content-Type"] = metadata.get("Content-Type", "application/octet-stream") for key, value in metadata.iteritems(): if is_user_meta("object", key) or key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata["ETag"] ts = metadata["X-Timestamp"] response.last_modified = math.ceil(float(ts)) # Needed for container sync feature response.headers["X-Timestamp"] = ts response.content_length = int(metadata["Content-Length"]) try: response.content_encoding = metadata["Content-Encoding"] except KeyError: pass return response @public @timing_stats() def DELETE(self, request): """Handle HTTP DELETE requests for the Swift Object Server.""" device, partition, account, container, obj = split_and_validate_path(request, 5, 5, True) if "x-timestamp" not in request.headers or not check_float(request.headers["x-timestamp"]): return HTTPBadRequest(body="Missing timestamp", request=request, content_type="text/plain") try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileExpired as e: orig_timestamp = e.timestamp orig_metadata = e.metadata response_class = HTTPNotFound except DiskFileDeleted as e: orig_timestamp = e.timestamp orig_metadata = {} response_class = HTTPNotFound except (DiskFileNotExist, DiskFileQuarantined): orig_timestamp = 0 orig_metadata = {} response_class = HTTPNotFound else: orig_timestamp = orig_metadata.get("X-Timestamp", 0) if orig_timestamp < request.headers["x-timestamp"]: response_class = HTTPNoContent else: response_class = HTTPConflict orig_delete_at = int(orig_metadata.get("X-Delete-At") or 0) try: req_if_delete_at_val = request.headers["x-if-delete-at"] req_if_delete_at = int(req_if_delete_at_val) except KeyError: pass except ValueError: return HTTPBadRequest(request=request, body="Bad X-If-Delete-At header value") else: if orig_delete_at != req_if_delete_at: return HTTPPreconditionFailed(request=request, body="X-If-Delete-At and X-Delete-At do not match") if orig_delete_at: self.delete_at_update("DELETE", orig_delete_at, account, container, obj, request, device) req_timestamp = request.headers["X-Timestamp"] if orig_timestamp < req_timestamp: disk_file.delete(req_timestamp) self.container_update( "DELETE", account, container, obj, request, HeaderKeyDict({"x-timestamp": req_timestamp}), device ) return response_class(request=request) @public @replication @timing_stats(sample_rate=0.1) def REPLICATE(self, request): """ Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. """ device, partition, suffix = split_and_validate_path(request, 2, 3, True) try: hashes = self._diskfile_mgr.get_hashes(device, partition, suffix) except DiskFileDeviceUnavailable: resp = HTTPInsufficientStorage(drive=device, request=request) else: resp = Response(body=pickle.dumps(hashes)) return resp @public @replication @timing_stats(sample_rate=0.1) def REPLICATION(self, request): return Response(app_iter=ssync_receiver.Receiver(self, request)()) def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get("x-trans-id", None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body="Invalid UTF8 or contains NULL") else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, "publicly_accessible") replication_method = getattr(method, "replication", False) if self.replication_server is not None and self.replication_server != replication_method: raise AttributeError("Not allowed method.") except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception( _("ERROR __call__ error with %(method)s" " %(path)s "), {"method": req.method, "path": req.path} ) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f' % ( req.remote_addr, time.strftime("%d/%b/%Y:%H:%M:%S +0000", time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or "-", req.referer or "-", req.headers.get("x-trans-id", "-"), req.user_agent or "-", trans_time, ) if req.method in ("REPLICATE", "REPLICATION") or "X-Backend-Replication" in req.headers: self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ("PUT", "DELETE"): slow = self.slow - trans_time if slow > 0: sleep(slow) return res(env, start_response)