def create(self, size=None):
"""
Context manager to create a file. We create a temporary file first, and
then return a DiskWriter object to encapsulate the state.
:param size: optional initial size of file to explicitly allocate on
disk
:raises DiskFileNoSpace: if a size is specified and allocation fails
"""
if not exists(self.tmpdir):
mkdirs(self.tmpdir)
fd, tmppath = mkstemp(dir=self.tmpdir)
try:
if size is not None and size > 0:
try:
fallocate(fd, size)
except OSError:
raise DiskFileNoSpace()
yield DiskWriter(self, fd, tmppath, self.threadpool)
finally:
try:
os.close(fd)
except OSError:
pass
try:
os.unlink(tmppath)
except OSError:
pass
def _preallocate(self):
"""
The idea is to allocate space in front of an expanding db. If it gets
within 512k of a boundary, it allocates to the next boundary.
Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after.
"""
if not DB_PREALLOCATION or self.db_file == ':memory:':
return
MB = (1024 * 1024)
def prealloc_points():
for pm in (1, 2, 5, 10, 25, 50):
yield pm * MB
while True:
pm += 50
yield pm * MB
stat = os.stat(self.db_file)
file_size = stat.st_size
allocated_size = stat.st_blocks * 512
for point in prealloc_points():
if file_size <= point - MB / 2:
prealloc_size = point
break
if allocated_size < prealloc_size:
with open(self.db_file, 'rb+') as fp:
fallocate(fp.fileno(), int(prealloc_size))
def writer(self, size=None):
"""
Context manager to write a file. We create a temporary file first, and
then return a DiskWriter object to encapsulate the state.
:param size: optional initial size of file to explicitly allocate on
disk
:raises DiskFileNoSpace: if a size is specified and allocation fails
"""
if not os.path.exists(self.tmpdir):
mkdirs(self.tmpdir)
fd, tmppath = mkstemp(dir=self.tmpdir)
try:
if size is not None and size > 0:
try:
fallocate(fd, size)
except OSError:
raise DiskFileNoSpace()
yield DiskWriter(self, fd, tmppath, self.threadpool)
finally:
try:
os.close(fd)
except OSError:
pass
try:
os.unlink(tmppath)
except OSError:
pass
def _preallocate(self):
"""
The idea is to allocate space in front of an expanding db. If it gets
within 512k of a boundary, it allocates to the next boundary.
Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after.
"""
if not DB_PREALLOCATION or self.db_file == ':memory:':
return
MB = (1024 * 1024)
def prealloc_points():
for pm in (1, 2, 5, 10, 25, 50):
yield pm * MB
while True:
pm += 50
yield pm * MB
stat = os.stat(self.db_file)
file_size = stat.st_size
allocated_size = stat.st_blocks * 512
for point in prealloc_points():
if file_size <= point - MB / 2:
prealloc_size = point
break
if allocated_size < prealloc_size:
with open(self.db_file, 'rb+') as fp:
fallocate(fp.fileno(), int(prealloc_size))
def create(self, size=None):
"""
Context manager to create a file. We create a temporary file first, and
then return a DiskFileWriter object to encapsulate the state.
.. note::
An implementation is not required to perform on-disk
preallocations even if the parameter is specified. But if it does
and it fails, it must raise a `DiskFileNoSpace` exception.
:param size: optional initial size of file to explicitly allocate on
disk
:raises DiskFileNoSpace: if a size is specified and allocation fails
"""
if not exists(self._tmpdir):
mkdirs(self._tmpdir)
fd, tmppath = mkstemp(dir=self._tmpdir)
try:
if size is not None and size > 0:
try:
fallocate(fd, size)
except OSError:
raise DiskFileNoSpace()
yield DiskFileWriter(self._name, self._datadir, fd, tmppath,
self._bytes_per_sync, self._threadpool)
finally:
try:
os.close(fd)
except OSError:
pass
try:
os.unlink(tmppath)
except OSError:
pass
def mkstemp(self, size=None):
"""
Contextmanager to make a temporary file.
:param size: optional initial size of file to allocate on disk
:raises DiskFileNoSpace: if a size is specified and fallocate fails
"""
if not os.path.exists(self.tmpdir):
mkdirs(self.tmpdir)
fd, self.tmppath = mkstemp(dir=self.tmpdir)
try:
if size is not None and size > 0:
try:
fallocate(fd, size)
except OSError:
raise DiskFileNoSpace()
yield fd
finally:
try:
os.close(fd)
except OSError:
pass
tmppath, self.tmppath = self.tmppath, None
try:
os.unlink(tmppath)
except OSError:
pass
def create(self, size=None):
"""
Context manager to create a file. We create a temporary file first, and
then return a DiskFileWriter object to encapsulate the state.
.. note::
An implementation is not required to perform on-disk
preallocations even if the parameter is specified. But if it does
and it fails, it must raise a `DiskFileNoSpace` exception.
:param size: optional initial size of file to explicitly allocate on
disk
:raises DiskFileNoSpace: if a size is specified and allocation fails
"""
if not exists(self._tmpdir):
mkdirs(self._tmpdir)
fd, tmppath = mkstemp(dir=self._tmpdir)
try:
if size is not None and size > 0:
try:
fallocate(fd, size)
except OSError:
raise DiskFileNoSpace()
yield DiskFileWriter(self._name, self._datadir, fd, tmppath,
self._bytes_per_sync, self._threadpool)
finally:
try:
os.close(fd)
except OSError:
pass
try:
os.unlink(tmppath)
except OSError:
pass
new_delete_at = int(request.headers.get('X-Delete-At') or 0)
if new_delete_at and new_delete_at < time.time():
self.logger.increment('PUT.errors')
return HTTPBadRequest(body='X-Delete-At in past', request=request,
content_type='text/plain')
file = DiskFile(self.devices, device, partition, account, container,
obj, self.logger, disk_chunk_size=self.disk_chunk_size)
orig_timestamp = file.metadata.get('X-Timestamp')
upload_expiration = time.time() + self.max_upload_time
etag = md5()
upload_size = 0
last_sync = 0
with file.mkstemp() as (fd, tmppath):
if 'content-length' in request.headers:
try:
fallocate(fd, int(request.headers['content-length']))
except OSError:
return HTTPInsufficientStorage(drive=device,
request=request)
reader = request.environ['wsgi.input'].read
for chunk in iter(lambda: reader(self.network_chunk_size), ''):
upload_size += len(chunk)
if time.time() > upload_expiration:
self.logger.increment('PUT.timeouts')
return HTTPRequestTimeout(request=request)
etag.update(chunk)
while chunk:
written = os.write(fd, chunk)
chunk = chunk[written:]
# For large files sync every 512MB (by default) written
if upload_size - last_sync >= self.bytes_per_sync:
def mkstemp(self, size=None):
"""
Contextmanager to make a temporary file, optionally of a specified
initial size.
For Gluster, we first optimistically create the temporary file using
the "rsync-friendly" .NAME.random naming. If we find that some path to
the file does not exist, we then create that path and then create the
temporary file again. If we get file name conflict, we'll retry using
different random suffixes 1,000 times before giving up.
"""
data_file = os.path.join(self.put_datadir, self._obj)
# Assume the full directory path exists to the file already, and
# construct the proper name for the temporary file.
for i in range(0, 1000):
tmpfile = '.' + self._obj + '.' + md5(self._obj +
str(random.random())).hexdigest()
tmppath = os.path.join(self.put_datadir, tmpfile)
try:
fd = do_open(tmppath,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
except GlusterFileSystemOSError as gerr:
if gerr.errno == errno.EEXIST:
# Retry with a different random number.
continue
if gerr.errno == errno.EIO:
# FIXME: Possible FUSE issue or race condition, let's
# sleep on it and retry the operation.
_random_sleep()
logging.warn("DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs", gerr)
continue
if gerr.errno != errno.ENOENT:
# FIXME: Other cases we should handle?
raise
if not self._obj_path:
# No directory hierarchy and the create failed telling us
# the container or volume directory does not exist. This
# could be a FUSE issue or some race condition, so let's
# sleep a bit and retry.
_random_sleep()
logging.warn("DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs", gerr)
continue
if i != 0:
# Got ENOENT after previously making the path. This could
# also be a FUSE issue or some race condition, nap and
# retry.
_random_sleep()
logging.warn("DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs" % gerr)
continue
# It looks like the path to the object does not already exist
self._create_dir_object(self._obj_path)
continue
else:
break
else:
# We failed after 1,000 attempts to create the temporary file.
raise DiskFileError('DiskFile.mkstemp(): failed to successfully'
' create a temporary file without running'
' into a name conflict after 1,000 attempts'
' for: %s' % (data_file,))
self.tmppath = tmppath
try:
# Ensure it is properly owned before we make it available.
do_fchown(fd, self.uid, self.gid)
if _preallocate and size:
# For XFS, fallocate() turns off speculative pre-allocation
# until a write is issued either to the last block of the file
# before the EOF or beyond the EOF. This means that we are
# less likely to fragment free space with pre-allocated
# extents that get truncated back to the known file size.
# However, this call also turns holes into allocated but
# unwritten extents, so that allocation occurs before the
# write, not during XFS writeback. This effectively defeats
# any allocation optimizations the filesystem can make at
# writeback time.
fallocate(fd, size)
yield fd
finally:
try:
do_close(fd)
except OSError:
pass
if self.tmppath:
tmppath, self.tmppath = self.tmppath, None
do_unlink(tmppath)
return error_response
new_delete_at = int(request.headers.get('X-Delete-At') or 0)
if new_delete_at and new_delete_at < time.time():
return HTTPBadRequest(body='X-Delete-At in past', request=request,
content_type='text/plain')
file = DiskFile(self.devices, device, partition, account, container,
obj, self.logger, disk_chunk_size=self.disk_chunk_size)
orig_timestamp = file.metadata.get('X-Timestamp')
upload_expiration = time.time() + self.max_upload_time
etag = md5()
upload_size = 0
last_sync = 0
elasped_time = 0
with file.mkstemp() as fd:
try:
fallocate(fd, int(request.headers.get('content-length', 0)))
except OSError:
return HTTPInsufficientStorage(drive=device, request=request)
reader = request.environ['wsgi.input'].read
for chunk in iter(lambda: reader(self.network_chunk_size), ''):
start_time = time.time()
upload_size += len(chunk)
if time.time() > upload_expiration:
self.logger.increment('PUT.timeouts')
return HTTPRequestTimeout(request=request)
etag.update(chunk)
while chunk:
written = os.write(fd, chunk)
chunk = chunk[written:]
# For large files sync every 512MB (by default) written
if upload_size - last_sync >= self.bytes_per_sync:
if 'x-timestamp' not in request.headers or \
not check_float(request.headers['x-timestamp']):
return HTTPBadRequest(body='Missing timestamp', request=request,
content_type='text/plain')
error_response = check_object_creation(request, obj)
if error_response:
return error_response
file = DiskFile(self.devices, device, partition, account, container,
obj, disk_chunk_size=self.disk_chunk_size)
upload_expiration = time.time() + self.max_upload_time
etag = md5()
upload_size = 0
last_sync = 0
with file.mkstemp() as (fd, tmppath):
if 'content-length' in request.headers:
fallocate(fd, int(request.headers['content-length']))
for chunk in iter(lambda: request.body_file.read(
self.network_chunk_size), ''):
upload_size += len(chunk)
if time.time() > upload_expiration:
return HTTPRequestTimeout(request=request)
etag.update(chunk)
while chunk:
written = os.write(fd, chunk)
chunk = chunk[written:]
# For large files sync every 512MB (by default) written
if upload_size - last_sync >= self.bytes_per_sync:
tpool.execute(os.fdatasync, fd)
drop_buffer_cache(fd, last_sync, upload_size - last_sync)
last_sync = upload_size
def create(self, size=None):
"""
Context manager to create a file. We create a temporary file first, and
then return a DiskFileWriter object to encapsulate the state.
For Gluster, we first optimistically create the temporary file using
the "rsync-friendly" .NAME.random naming. If we find that some path to
the file does not exist, we then create that path and then create the
temporary file again. If we get file name conflict, we'll retry using
different random suffixes 1,000 times before giving up.
.. note::
An implementation is not required to perform on-disk
preallocations even if the parameter is specified. But if it does
and it fails, it must raise a `DiskFileNoSpace` exception.
:param size: optional initial size of file to explicitly allocate on
disk
:raises DiskFileNoSpace: if a size is specified and allocation fails
:raises AlreadyExistsAsFile: if path or part of a path is not a \
directory
"""
# Create /account/container directory structure on mount point root
try:
os.makedirs(self._container_path)
except OSError as err:
if err.errno != errno.EEXIST:
raise
data_file = os.path.join(self._put_datadir, self._obj)
# Assume the full directory path exists to the file already, and
# construct the proper name for the temporary file.
attempts = 1
while True:
# To know more about why following temp file naming convention is
# used, please read this GlusterFS doc:
# https://github.com/gluster/glusterfs/blob/master/doc/features/dht.md#rename-optimizations # noqa
tmpfile = '.' + self._obj + '.' + uuid4().hex
tmppath = os.path.join(self._put_datadir, tmpfile)
try:
fd = do_open(tmppath,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
except SwiftOnFileSystemOSError as gerr:
if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
# Raise DiskFileNoSpace to be handled by upper layers when
# there is no space on disk OR when quota is exceeded
raise DiskFileNoSpace()
if gerr.errno == errno.ENOTDIR:
raise AlreadyExistsAsFile('do_open(): failed on %s,'
' path or part of a'
' path is not a directory' %
(tmppath))
if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
# FIXME: Other cases we should handle?
raise
if attempts >= MAX_OPEN_ATTEMPTS:
# We failed after N attempts to create the temporary
# file.
raise DiskFileError(
'DiskFile.mkstemp(): failed to'
' successfully create a temporary file'
' without running into a name conflict'
' after %d of %d attempts for: %s' %
(attempts, MAX_OPEN_ATTEMPTS, data_file))
if gerr.errno == errno.EEXIST:
# Retry with a different random number.
attempts += 1
elif gerr.errno == errno.EIO:
# FIXME: Possible FUSE issue or race condition, let's
# sleep on it and retry the operation.
_random_sleep()
logging.warn(
"DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs", gerr)
attempts += 1
elif not self._obj_path:
# No directory hierarchy and the create failed telling us
# the container or volume directory does not exist. This
# could be a FUSE issue or some race condition, so let's
# sleep a bit and retry.
_random_sleep()
logging.warn(
"DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs", gerr)
attempts += 1
elif attempts > 1:
# Got ENOENT after previously making the path. This could
# also be a FUSE issue or some race condition, nap and
# retry.
_random_sleep()
logging.warn("DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs" % gerr)
attempts += 1
else:
# It looks like the path to the object does not already
# exist; don't count this as an attempt, though, since
# we perform the open() system call optimistically.
self._create_dir_object(self._obj_path)
else:
break
dw = None
try:
if size is not None and size > 0:
try:
fallocate(fd, size)
except OSError as err:
if err.errno in (errno.ENOSPC, errno.EDQUOT):
raise DiskFileNoSpace()
raise
# Ensure it is properly owned before we make it available.
do_fchown(fd, self._uid, self._gid)
dw = DiskFileWriter(fd, tmppath, self)
yield dw
finally:
if dw:
dw.close()
if dw._tmppath:
do_unlink(dw._tmppath)
return error_response
new_delete_at = int(request.headers.get("X-Delete-At") or 0)
if new_delete_at and new_delete_at < time.time():
return HTTPBadRequest(body="X-Delete-At in past", request=request, content_type="text/plain")
file = DiskFile(
self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size
)
orig_timestamp = file.metadata.get("X-Timestamp")
upload_expiration = time.time() + self.max_upload_time
etag = md5()
upload_size = 0
last_sync = 0
with file.mkstemp() as (fd, tmppath):
if "content-length" in request.headers:
try:
fallocate(fd, int(request.headers["content-length"]))
except OSError:
return HTTPInsufficientStorage(drive=device, request=request)
reader = request.environ["wsgi.input"].read
for chunk in iter(lambda: reader(self.network_chunk_size), ""):
upload_size += len(chunk)
if time.time() > upload_expiration:
self.logger.increment("PUT.timeouts")
return HTTPRequestTimeout(request=request)
etag.update(chunk)
while chunk:
written = os.write(fd, chunk)
chunk = chunk[written:]
# For large files sync every 512MB (by default) written
if upload_size - last_sync >= self.bytes_per_sync:
tpool.execute(fsync, fd)
def create(self, size=None):
"""
Context manager to create a file. We create a temporary file first, and
then return a DiskFileWriter object to encapsulate the state.
For Gluster, we first optimistically create the temporary file using
the "rsync-friendly" .NAME.random naming. If we find that some path to
the file does not exist, we then create that path and then create the
temporary file again. If we get file name conflict, we'll retry using
different random suffixes 1,000 times before giving up.
.. note::
An implementation is not required to perform on-disk
preallocations even if the parameter is specified. But if it does
and it fails, it must raise a `DiskFileNoSpace` exception.
:param size: optional initial size of file to explicitly allocate on
disk
:raises DiskFileNoSpace: if a size is specified and allocation fails
:raises AlreadyExistsAsFile: if path or part of a path is not a \
directory
"""
# Create /account/container directory structure on mount point root
try:
os.makedirs(self._container_path)
except OSError as err:
if err.errno != errno.EEXIST:
raise
data_file = os.path.join(self._put_datadir, self._obj)
# Assume the full directory path exists to the file already, and
# construct the proper name for the temporary file.
attempts = 1
while True:
# To know more about why following temp file naming convention is
# used, please read this GlusterFS doc:
# https://github.com/gluster/glusterfs/blob/master/doc/features/dht.md#rename-optimizations # noqa
tmpfile = '.' + self._obj + '.' + uuid4().hex
tmppath = os.path.join(self._put_datadir, tmpfile)
try:
fd = do_open(tmppath,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
except SwiftOnFileSystemOSError as gerr:
if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
# Raise DiskFileNoSpace to be handled by upper layers when
# there is no space on disk OR when quota is exceeded
raise DiskFileNoSpace()
if gerr.errno == errno.ENOTDIR:
raise AlreadyExistsAsFile('do_open(): failed on %s,'
' path or part of a'
' path is not a directory'
% (tmppath))
if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
# FIXME: Other cases we should handle?
raise
if attempts >= MAX_OPEN_ATTEMPTS:
# We failed after N attempts to create the temporary
# file.
raise DiskFileError('DiskFile.mkstemp(): failed to'
' successfully create a temporary file'
' without running into a name conflict'
' after %d of %d attempts for: %s' % (
attempts, MAX_OPEN_ATTEMPTS,
data_file))
if gerr.errno == errno.EEXIST:
# Retry with a different random number.
attempts += 1
elif gerr.errno == errno.EIO:
# FIXME: Possible FUSE issue or race condition, let's
# sleep on it and retry the operation.
_random_sleep()
logging.warn("DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs", gerr)
attempts += 1
elif not self._obj_path:
# No directory hierarchy and the create failed telling us
# the container or volume directory does not exist. This
# could be a FUSE issue or some race condition, so let's
# sleep a bit and retry.
_random_sleep()
logging.warn("DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs", gerr)
attempts += 1
elif attempts > 1:
# Got ENOENT after previously making the path. This could
# also be a FUSE issue or some race condition, nap and
# retry.
_random_sleep()
logging.warn("DiskFile.mkstemp(): %s ... retrying in"
" 0.1 secs" % gerr)
attempts += 1
else:
# It looks like the path to the object does not already
# exist; don't count this as an attempt, though, since
# we perform the open() system call optimistically.
self._create_dir_object(self._obj_path)
else:
break
dw = None
try:
if size is not None and size > 0:
try:
fallocate(fd, size)
except OSError as err:
if err.errno in (errno.ENOSPC, errno.EDQUOT):
raise DiskFileNoSpace()
raise
# Ensure it is properly owned before we make it available.
do_fchown(fd, self._uid, self._gid)
dw = DiskFileWriter(fd, tmppath, self)
yield dw
finally:
if dw:
dw.close()
if dw._tmppath:
do_unlink(dw._tmppath)
device,
partition,
account,
container,
obj,
self.logger,
disk_chunk_size=self.disk_chunk_size)
orig_timestamp = file.metadata.get('X-Timestamp')
upload_expiration = time.time() + self.max_upload_time
etag = md5()
upload_size = 0
last_sync = 0
elapsed_time = 0
with file.mkstemp() as fd:
try:
fallocate(fd, int(request.headers.get('content-length', 0)))
except OSError:
return HTTPInsufficientStorage(drive=device, request=request)
reader = request.environ['wsgi.input'].read
for chunk in iter(lambda: reader(self.network_chunk_size), ''):
start_time = time.time()
upload_size += len(chunk)
if time.time() > upload_expiration:
self.logger.increment('PUT.timeouts')
return HTTPRequestTimeout(request=request)
etag.update(chunk)
while chunk:
written = os.write(fd, chunk)
chunk = chunk[written:]
# For large files sync every 512MB (by default) written
if upload_size - last_sync >= self.bytes_per_sync: