def test_check_object_creation(self): req = Mock() req.headers = dict() valid_object_names = ["a/b/c/d", '/'.join(("1@3%&*0-", "};+=]|")), '/'.join(('a' * 255, 'b' * 255, 'c' * 221))] for o in valid_object_names: self.assertFalse(cnt.check_object_creation(req, o)) invalid_object_names = ["a/./b", "a/b/../d", "a//b", "a/c//", '/'.join(('a' * 256, 'b' * 255, 'c' * 221)), '/'.join(('a' * 255, 'b' * 255, 'c' * 222))] for o in invalid_object_names: self.assertTrue(cnt.check_object_creation(req, o)) # Check for creation of directory marker objects that ends with slash with patch.dict(req.headers, {'content-type': 'application/directory'}): self.assertFalse(cnt.check_object_creation(req, "a/b/c/d/")) # Check creation of objects ending with slash having any other content # type than application/directory is not allowed for content_type in ('text/plain', 'text/html', 'image/jpg', 'application/octet-stream', 'blah/blah'): with patch.dict(req.headers, {'content-type': content_type}): self.assertTrue(cnt.check_object_creation(req, "a/b/c/d/"))
def PUT(self, request): """Handle HTTP PUT requests for the Swift on File object server""" try: device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) # check swiftonhpss constraints first error_response = check_object_creation(request, obj) if error_response: return error_response # (HPSS) Shameless copy-paste from ObjectController.PUT and # modification, because we have to do certain things like pass in # purgelock and class-of-service information that Swift won't know # to do and need to do it in a very specific order. new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, context_type='text/plain') try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') # Try to get DiskFile try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy=policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} # Check for If-None-Match in request if request.if_none_match and orig_metadata: if '*' in request.if_none_match: # File exists already, return 412 return HTTPPreconditionFailed(request=request) if orig_metadata.get('ETag') in request.if_none_match: # The current ETag matches, return 412 return HTTPPreconditionFailed(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: return HTTPConflict( request=request, headers={'X-Backend-Timestamp': orig_timestamp.internal}) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 # (HPSS) Check for HPSS-specific metadata headers cos = request.headers.get('X-Hpss-Class-Of-Service-Id', None) purgelock = config_true_value( request.headers.get('X-Hpss-Purgelock-Status', 'false')) try: # Feed DiskFile our HPSS-specific stuff with disk_file.create(size=fsize, cos=cos) as writer: upload_size = 0 def timeout_reader(): with ChunkReadTimeout(self.client_timeout): return request.environ['wsgi.input'].read( self.network_chunk_size) try: for chunk in iter(lambda: timeout_reader(), ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time except ChunkReadTimeout: return HTTPRequestTimeout(request=request) if upload_size: self.logger.transfer_rate('PUT.%s.timing' % device, elapsed_time, upload_size) if fsize and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if 'etag' in request.headers \ and request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) # Update object metadata metadata = {'X-Timestamp': request.timestamp.internal, 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), } meta_headers = {header: request.headers[header] for header in request.headers if is_sys_or_user_meta('object', header)} metadata.update(meta_headers) backend_headers = \ request.headers.get('X-Backend-Replication-Headers') for header_key in (backend_headers or self.allowed_headers): if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] # (HPSS) Write the file, with added options writer.put(metadata, purgelock=purgelock) except DiskFileNoSpace: return HTTPInsufficientStorage(drive=device, request=request) except SwiftOnFileSystemIOError as e: return HTTPServiceUnavailable(request=request) # FIXME: this stuff really should be handled in DiskFile somehow? # we set the hpss checksum in here, so both systems have valid # and current checksum metadata # (HPSS) Set checksum on file ourselves, if hpssfs won't do it # for us. data_file = disk_file._data_file try: xattr.setxattr(data_file, 'system.hpss.hash', "md5:%s" % etag) except IOError: logging.debug("Could not write ETag to system.hpss.hash," " trying user.hash.checksum") try: xattr.setxattr(data_file, 'user.hash.checksum', etag) xattr.setxattr(data_file, 'user.hash.algorithm', 'md5') xattr.setxattr(data_file, 'user.hash.state', 'Valid') xattr.setxattr(data_file, 'user.hash.filesize', str(upload_size)) xattr.setxattr(data_file, 'user.hash.app', 'swiftonhpss') except IOError as err: raise SwiftOnFileSystemIOError( err.errno, 'Could not write MD5 checksum to HPSS filesystem: ' '%s' % err.strerror) # Update container metadata if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device, policy) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy) self.container_update('PUT', account, container, obj, request, HeaderKeyDict( {'x-size': metadata['Content-Length'], 'x-content-type': metadata['Content-Type'], 'x-timestamp': metadata['X-Timestamp'], 'x-etag': metadata['ETag']}), device, policy) # Create convenience symlink try: self._object_symlink(request, disk_file._data_file, device, account) except SwiftOnFileSystemOSError: logging.debug('could not make account symlink') return HTTPServiceUnavailable(request=request) return HTTPCreated(request=request, etag=etag) except (AlreadyExistsAsFile, AlreadyExistsAsDir): device = \ split_and_validate_path(request, 1, 5, True) return HTTPConflict(drive=device, request=request)
def PUT(self, request): """Handle HTTP PUT requests for the Swift on File object server""" try: device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) # check swiftonhpss constraints first error_response = check_object_creation(request, obj) if error_response: return error_response # (HPSS) Shameless copy-paste from ObjectController.PUT and # modification, because we have to do certain things like pass in # purgelock and class-of-service information that Swift won't know # to do and need to do it in a very specific order. new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, context_type='text/plain') try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') self.logger.debug("DiskFile @ %s/%s/%s/%s" % (device, account, container, obj)) # Try to get DiskFile try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy=policy, uid=int(self.hpss_uid), gid=int(self.hpss_gid)) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} # Check for If-None-Match in request if request.if_none_match and orig_metadata: if '*' in request.if_none_match: # File exists already, return 412 return HTTPPreconditionFailed(request=request) if orig_metadata.get('ETag') in request.if_none_match: # The current ETag matches, return 412 return HTTPPreconditionFailed(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: return HTTPConflict( request=request, headers={'X-Backend-Timestamp': orig_timestamp.internal}) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time self.logger.debug("Receiving and writing object") etag = md5() elapsed_time = 0 hints = {'cos': request.headers.get('X-Hpss-Class-Of-Service-Id', self.default_cos_id), 'purgelock': request.headers.get('X-Hpss-Purgelock-Status', False)} if request.headers['content-type'] == 'application/directory': # TODO: handle directories different pass try: # Feed DiskFile our HPSS-specific stuff with disk_file.create(hpss_hints=hints) as writer: upload_size = 0 def timeout_reader(): with ChunkReadTimeout(self.client_timeout): return request.environ['wsgi.input'].read( self.network_chunk_size) try: for chunk in iter(lambda: timeout_reader(), ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time except ChunkReadTimeout: return HTTPRequestTimeout(request=request) if upload_size: self.logger.transfer_rate('PUT.%s.timing' % device, elapsed_time, upload_size) if fsize and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if 'etag' in request.headers \ and request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) self.logger.debug("Writing object metadata") # Update object metadata content_type = request.headers['content-type'] metadata = {'X-Timestamp': request.timestamp.internal, 'Content-Type': content_type, 'ETag': etag, 'Content-Length': str(upload_size), } meta_headers = {header: request.headers[header] for header in request.headers if is_sys_or_user_meta('object', header)} metadata.update(meta_headers) backend_headers = \ request.headers.get('X-Backend-Replication-Headers') for header_key in (backend_headers or self.allowed_headers): if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] self.logger.debug("Finalizing object") # (HPSS) Write the file, with added options writer.put(metadata) except DiskFileNoSpace: return HTTPInsufficientStorage(drive=device, request=request) except SwiftOnFileSystemIOError as e: self.logger.error(e) return HTTPServiceUnavailable(request=request) self.logger.debug("Writing container metadata") # Update container metadata if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device, policy) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy) container_headers = {'x-size': metadata['Content-Length'], 'x-content-type': metadata['Content-Type'], 'x-timestamp': metadata['X-Timestamp'], 'x-etag': metadata['ETag']} self.container_update('PUT', account, container, obj, request, HeaderKeyDict(container_headers), device, policy) self.logger.debug("Done!") # Create convenience symlink try: self._project_symlink(request, disk_file, account) except SwiftOnFileSystemOSError: logging.debug('could not make account symlink') return HTTPServiceUnavailable(request=request) return HTTPCreated(request=request, etag=etag) except (AlreadyExistsAsFile, AlreadyExistsAsDir): device = \ split_and_validate_path(request, 1, 5, True) return HTTPConflict(drive=device, request=request)