def create(self, size=None): """ Context manager to create a file. We create a temporary file first, and then return a DiskWriter object to encapsulate the state. :param size: optional initial size of file to explicitly allocate on disk :raises DiskFileNoSpace: if a size is specified and allocation fails """ if not exists(self.tmpdir): mkdirs(self.tmpdir) fd, tmppath = mkstemp(dir=self.tmpdir) try: if size is not None and size > 0: try: fallocate(fd, size) except OSError: raise DiskFileNoSpace() yield DiskWriter(self, fd, tmppath, self.threadpool) finally: try: os.close(fd) except OSError: pass try: os.unlink(tmppath) except OSError: pass
def _preallocate(self): """ The idea is to allocate space in front of an expanding db. If it gets within 512k of a boundary, it allocates to the next boundary. Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after. """ if not DB_PREALLOCATION or self.db_file == ':memory:': return MB = (1024 * 1024) def prealloc_points(): for pm in (1, 2, 5, 10, 25, 50): yield pm * MB while True: pm += 50 yield pm * MB stat = os.stat(self.db_file) file_size = stat.st_size allocated_size = stat.st_blocks * 512 for point in prealloc_points(): if file_size <= point - MB / 2: prealloc_size = point break if allocated_size < prealloc_size: with open(self.db_file, 'rb+') as fp: fallocate(fp.fileno(), int(prealloc_size))
def writer(self, size=None): """ Context manager to write a file. We create a temporary file first, and then return a DiskWriter object to encapsulate the state. :param size: optional initial size of file to explicitly allocate on disk :raises DiskFileNoSpace: if a size is specified and allocation fails """ if not os.path.exists(self.tmpdir): mkdirs(self.tmpdir) fd, tmppath = mkstemp(dir=self.tmpdir) try: if size is not None and size > 0: try: fallocate(fd, size) except OSError: raise DiskFileNoSpace() yield DiskWriter(self, fd, tmppath, self.threadpool) finally: try: os.close(fd) except OSError: pass try: os.unlink(tmppath) except OSError: pass
def create(self, size=None): """ Context manager to create a file. We create a temporary file first, and then return a DiskFileWriter object to encapsulate the state. .. note:: An implementation is not required to perform on-disk preallocations even if the parameter is specified. But if it does and it fails, it must raise a `DiskFileNoSpace` exception. :param size: optional initial size of file to explicitly allocate on disk :raises DiskFileNoSpace: if a size is specified and allocation fails """ if not exists(self._tmpdir): mkdirs(self._tmpdir) fd, tmppath = mkstemp(dir=self._tmpdir) try: if size is not None and size > 0: try: fallocate(fd, size) except OSError: raise DiskFileNoSpace() yield DiskFileWriter(self._name, self._datadir, fd, tmppath, self._bytes_per_sync, self._threadpool) finally: try: os.close(fd) except OSError: pass try: os.unlink(tmppath) except OSError: pass
def mkstemp(self, size=None): """ Contextmanager to make a temporary file. :param size: optional initial size of file to allocate on disk :raises DiskFileNoSpace: if a size is specified and fallocate fails """ if not os.path.exists(self.tmpdir): mkdirs(self.tmpdir) fd, self.tmppath = mkstemp(dir=self.tmpdir) try: if size is not None and size > 0: try: fallocate(fd, size) except OSError: raise DiskFileNoSpace() yield fd finally: try: os.close(fd) except OSError: pass tmppath, self.tmppath = self.tmppath, None try: os.unlink(tmppath) except OSError: pass
new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): self.logger.increment('PUT.errors') return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') file = DiskFile(self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size) orig_timestamp = file.metadata.get('X-Timestamp') upload_expiration = time.time() + self.max_upload_time etag = md5() upload_size = 0 last_sync = 0 with file.mkstemp() as (fd, tmppath): if 'content-length' in request.headers: try: fallocate(fd, int(request.headers['content-length'])) except OSError: return HTTPInsufficientStorage(drive=device, request=request) reader = request.environ['wsgi.input'].read for chunk in iter(lambda: reader(self.network_chunk_size), ''): upload_size += len(chunk) if time.time() > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) while chunk: written = os.write(fd, chunk) chunk = chunk[written:] # For large files sync every 512MB (by default) written if upload_size - last_sync >= self.bytes_per_sync:
def mkstemp(self, size=None): """ Contextmanager to make a temporary file, optionally of a specified initial size. For Gluster, we first optimistically create the temporary file using the "rsync-friendly" .NAME.random naming. If we find that some path to the file does not exist, we then create that path and then create the temporary file again. If we get file name conflict, we'll retry using different random suffixes 1,000 times before giving up. """ data_file = os.path.join(self.put_datadir, self._obj) # Assume the full directory path exists to the file already, and # construct the proper name for the temporary file. for i in range(0, 1000): tmpfile = '.' + self._obj + '.' + md5(self._obj + str(random.random())).hexdigest() tmppath = os.path.join(self.put_datadir, tmpfile) try: fd = do_open(tmppath, os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC) except GlusterFileSystemOSError as gerr: if gerr.errno == errno.EEXIST: # Retry with a different random number. continue if gerr.errno == errno.EIO: # FIXME: Possible FUSE issue or race condition, let's # sleep on it and retry the operation. _random_sleep() logging.warn("DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs", gerr) continue if gerr.errno != errno.ENOENT: # FIXME: Other cases we should handle? raise if not self._obj_path: # No directory hierarchy and the create failed telling us # the container or volume directory does not exist. This # could be a FUSE issue or some race condition, so let's # sleep a bit and retry. _random_sleep() logging.warn("DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs", gerr) continue if i != 0: # Got ENOENT after previously making the path. This could # also be a FUSE issue or some race condition, nap and # retry. _random_sleep() logging.warn("DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs" % gerr) continue # It looks like the path to the object does not already exist self._create_dir_object(self._obj_path) continue else: break else: # We failed after 1,000 attempts to create the temporary file. raise DiskFileError('DiskFile.mkstemp(): failed to successfully' ' create a temporary file without running' ' into a name conflict after 1,000 attempts' ' for: %s' % (data_file,)) self.tmppath = tmppath try: # Ensure it is properly owned before we make it available. do_fchown(fd, self.uid, self.gid) if _preallocate and size: # For XFS, fallocate() turns off speculative pre-allocation # until a write is issued either to the last block of the file # before the EOF or beyond the EOF. This means that we are # less likely to fragment free space with pre-allocated # extents that get truncated back to the known file size. # However, this call also turns holes into allocated but # unwritten extents, so that allocation occurs before the # write, not during XFS writeback. This effectively defeats # any allocation optimizations the filesystem can make at # writeback time. fallocate(fd, size) yield fd finally: try: do_close(fd) except OSError: pass if self.tmppath: tmppath, self.tmppath = self.tmppath, None do_unlink(tmppath)
return error_response new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') file = DiskFile(self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size) orig_timestamp = file.metadata.get('X-Timestamp') upload_expiration = time.time() + self.max_upload_time etag = md5() upload_size = 0 last_sync = 0 elasped_time = 0 with file.mkstemp() as fd: try: fallocate(fd, int(request.headers.get('content-length', 0))) except OSError: return HTTPInsufficientStorage(drive=device, request=request) reader = request.environ['wsgi.input'].read for chunk in iter(lambda: reader(self.network_chunk_size), ''): start_time = time.time() upload_size += len(chunk) if time.time() > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) while chunk: written = os.write(fd, chunk) chunk = chunk[written:] # For large files sync every 512MB (by default) written if upload_size - last_sync >= self.bytes_per_sync:
if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') error_response = check_object_creation(request, obj) if error_response: return error_response file = DiskFile(self.devices, device, partition, account, container, obj, disk_chunk_size=self.disk_chunk_size) upload_expiration = time.time() + self.max_upload_time etag = md5() upload_size = 0 last_sync = 0 with file.mkstemp() as (fd, tmppath): if 'content-length' in request.headers: fallocate(fd, int(request.headers['content-length'])) for chunk in iter(lambda: request.body_file.read( self.network_chunk_size), ''): upload_size += len(chunk) if time.time() > upload_expiration: return HTTPRequestTimeout(request=request) etag.update(chunk) while chunk: written = os.write(fd, chunk) chunk = chunk[written:] # For large files sync every 512MB (by default) written if upload_size - last_sync >= self.bytes_per_sync: tpool.execute(os.fdatasync, fd) drop_buffer_cache(fd, last_sync, upload_size - last_sync) last_sync = upload_size
def create(self, size=None): """ Context manager to create a file. We create a temporary file first, and then return a DiskFileWriter object to encapsulate the state. For Gluster, we first optimistically create the temporary file using the "rsync-friendly" .NAME.random naming. If we find that some path to the file does not exist, we then create that path and then create the temporary file again. If we get file name conflict, we'll retry using different random suffixes 1,000 times before giving up. .. note:: An implementation is not required to perform on-disk preallocations even if the parameter is specified. But if it does and it fails, it must raise a `DiskFileNoSpace` exception. :param size: optional initial size of file to explicitly allocate on disk :raises DiskFileNoSpace: if a size is specified and allocation fails :raises AlreadyExistsAsFile: if path or part of a path is not a \ directory """ # Create /account/container directory structure on mount point root try: os.makedirs(self._container_path) except OSError as err: if err.errno != errno.EEXIST: raise data_file = os.path.join(self._put_datadir, self._obj) # Assume the full directory path exists to the file already, and # construct the proper name for the temporary file. attempts = 1 while True: # To know more about why following temp file naming convention is # used, please read this GlusterFS doc: # https://github.com/gluster/glusterfs/blob/master/doc/features/dht.md#rename-optimizations # noqa tmpfile = '.' + self._obj + '.' + uuid4().hex tmppath = os.path.join(self._put_datadir, tmpfile) try: fd = do_open(tmppath, os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC) except SwiftOnFileSystemOSError as gerr: if gerr.errno in (errno.ENOSPC, errno.EDQUOT): # Raise DiskFileNoSpace to be handled by upper layers when # there is no space on disk OR when quota is exceeded raise DiskFileNoSpace() if gerr.errno == errno.ENOTDIR: raise AlreadyExistsAsFile('do_open(): failed on %s,' ' path or part of a' ' path is not a directory' % (tmppath)) if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO): # FIXME: Other cases we should handle? raise if attempts >= MAX_OPEN_ATTEMPTS: # We failed after N attempts to create the temporary # file. raise DiskFileError( 'DiskFile.mkstemp(): failed to' ' successfully create a temporary file' ' without running into a name conflict' ' after %d of %d attempts for: %s' % (attempts, MAX_OPEN_ATTEMPTS, data_file)) if gerr.errno == errno.EEXIST: # Retry with a different random number. attempts += 1 elif gerr.errno == errno.EIO: # FIXME: Possible FUSE issue or race condition, let's # sleep on it and retry the operation. _random_sleep() logging.warn( "DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs", gerr) attempts += 1 elif not self._obj_path: # No directory hierarchy and the create failed telling us # the container or volume directory does not exist. This # could be a FUSE issue or some race condition, so let's # sleep a bit and retry. _random_sleep() logging.warn( "DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs", gerr) attempts += 1 elif attempts > 1: # Got ENOENT after previously making the path. This could # also be a FUSE issue or some race condition, nap and # retry. _random_sleep() logging.warn("DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs" % gerr) attempts += 1 else: # It looks like the path to the object does not already # exist; don't count this as an attempt, though, since # we perform the open() system call optimistically. self._create_dir_object(self._obj_path) else: break dw = None try: if size is not None and size > 0: try: fallocate(fd, size) except OSError as err: if err.errno in (errno.ENOSPC, errno.EDQUOT): raise DiskFileNoSpace() raise # Ensure it is properly owned before we make it available. do_fchown(fd, self._uid, self._gid) dw = DiskFileWriter(fd, tmppath, self) yield dw finally: if dw: dw.close() if dw._tmppath: do_unlink(dw._tmppath)
return error_response new_delete_at = int(request.headers.get("X-Delete-At") or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body="X-Delete-At in past", request=request, content_type="text/plain") file = DiskFile( self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size ) orig_timestamp = file.metadata.get("X-Timestamp") upload_expiration = time.time() + self.max_upload_time etag = md5() upload_size = 0 last_sync = 0 with file.mkstemp() as (fd, tmppath): if "content-length" in request.headers: try: fallocate(fd, int(request.headers["content-length"])) except OSError: return HTTPInsufficientStorage(drive=device, request=request) reader = request.environ["wsgi.input"].read for chunk in iter(lambda: reader(self.network_chunk_size), ""): upload_size += len(chunk) if time.time() > upload_expiration: self.logger.increment("PUT.timeouts") return HTTPRequestTimeout(request=request) etag.update(chunk) while chunk: written = os.write(fd, chunk) chunk = chunk[written:] # For large files sync every 512MB (by default) written if upload_size - last_sync >= self.bytes_per_sync: tpool.execute(fsync, fd)
def create(self, size=None): """ Context manager to create a file. We create a temporary file first, and then return a DiskFileWriter object to encapsulate the state. For Gluster, we first optimistically create the temporary file using the "rsync-friendly" .NAME.random naming. If we find that some path to the file does not exist, we then create that path and then create the temporary file again. If we get file name conflict, we'll retry using different random suffixes 1,000 times before giving up. .. note:: An implementation is not required to perform on-disk preallocations even if the parameter is specified. But if it does and it fails, it must raise a `DiskFileNoSpace` exception. :param size: optional initial size of file to explicitly allocate on disk :raises DiskFileNoSpace: if a size is specified and allocation fails :raises AlreadyExistsAsFile: if path or part of a path is not a \ directory """ # Create /account/container directory structure on mount point root try: os.makedirs(self._container_path) except OSError as err: if err.errno != errno.EEXIST: raise data_file = os.path.join(self._put_datadir, self._obj) # Assume the full directory path exists to the file already, and # construct the proper name for the temporary file. attempts = 1 while True: # To know more about why following temp file naming convention is # used, please read this GlusterFS doc: # https://github.com/gluster/glusterfs/blob/master/doc/features/dht.md#rename-optimizations # noqa tmpfile = '.' + self._obj + '.' + uuid4().hex tmppath = os.path.join(self._put_datadir, tmpfile) try: fd = do_open(tmppath, os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC) except SwiftOnFileSystemOSError as gerr: if gerr.errno in (errno.ENOSPC, errno.EDQUOT): # Raise DiskFileNoSpace to be handled by upper layers when # there is no space on disk OR when quota is exceeded raise DiskFileNoSpace() if gerr.errno == errno.ENOTDIR: raise AlreadyExistsAsFile('do_open(): failed on %s,' ' path or part of a' ' path is not a directory' % (tmppath)) if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO): # FIXME: Other cases we should handle? raise if attempts >= MAX_OPEN_ATTEMPTS: # We failed after N attempts to create the temporary # file. raise DiskFileError('DiskFile.mkstemp(): failed to' ' successfully create a temporary file' ' without running into a name conflict' ' after %d of %d attempts for: %s' % ( attempts, MAX_OPEN_ATTEMPTS, data_file)) if gerr.errno == errno.EEXIST: # Retry with a different random number. attempts += 1 elif gerr.errno == errno.EIO: # FIXME: Possible FUSE issue or race condition, let's # sleep on it and retry the operation. _random_sleep() logging.warn("DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs", gerr) attempts += 1 elif not self._obj_path: # No directory hierarchy and the create failed telling us # the container or volume directory does not exist. This # could be a FUSE issue or some race condition, so let's # sleep a bit and retry. _random_sleep() logging.warn("DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs", gerr) attempts += 1 elif attempts > 1: # Got ENOENT after previously making the path. This could # also be a FUSE issue or some race condition, nap and # retry. _random_sleep() logging.warn("DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs" % gerr) attempts += 1 else: # It looks like the path to the object does not already # exist; don't count this as an attempt, though, since # we perform the open() system call optimistically. self._create_dir_object(self._obj_path) else: break dw = None try: if size is not None and size > 0: try: fallocate(fd, size) except OSError as err: if err.errno in (errno.ENOSPC, errno.EDQUOT): raise DiskFileNoSpace() raise # Ensure it is properly owned before we make it available. do_fchown(fd, self._uid, self._gid) dw = DiskFileWriter(fd, tmppath, self) yield dw finally: if dw: dw.close() if dw._tmppath: do_unlink(dw._tmppath)
device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size) orig_timestamp = file.metadata.get('X-Timestamp') upload_expiration = time.time() + self.max_upload_time etag = md5() upload_size = 0 last_sync = 0 elapsed_time = 0 with file.mkstemp() as fd: try: fallocate(fd, int(request.headers.get('content-length', 0))) except OSError: return HTTPInsufficientStorage(drive=device, request=request) reader = request.environ['wsgi.input'].read for chunk in iter(lambda: reader(self.network_chunk_size), ''): start_time = time.time() upload_size += len(chunk) if time.time() > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) while chunk: written = os.write(fd, chunk) chunk = chunk[written:] # For large files sync every 512MB (by default) written if upload_size - last_sync >= self.bytes_per_sync: