def _create_file(self, filename, size, sparse=True):
try:
f = open(filename, "w+")
except (OSError, IOError):
raise ExecutionError("Could not open %s" % filename)
try:
if sparse:
try:
os.posix_fallocate(f.fileno(), 0, size)
except AttributeError:
# Prior to version 3.3, Python does not provide fallocate
os.ftruncate(f.fileno(), size)
else:
self.shell.log.info("Writing %d bytes" % size)
while size > 0:
write_size = min(size, 1024)
f.write("\0" * write_size)
size -= write_size
except (OSError, IOError):
os.remove(filename)
raise ExecutionError("Could not expand file to %d bytes" % size)
except OverflowError:
raise ExecutionError("The file size is too large (%d bytes)" %
size)
finally:
f.close()
def _allocate_disk_space(self):
# type: (Descriptor) -> None
"""Perform file allocation (possibly sparse)
:param Descriptor self: this
"""
with self._meta_lock:
if self._allocated or self._offset != 0:
return
# set local path view
allocatesize = self._set_final_path_view()
# check if path already exists and is of sufficient size
if (not self.final_path.exists() or
self.final_path.stat().st_size != allocatesize):
# create parent path
self.final_path.parent.mkdir(
mode=0o750, parents=True, exist_ok=True)
# allocate file
with self.final_path.open('wb') as fd:
if allocatesize > 0:
try:
os.posix_fallocate(fd.fileno(), 0, allocatesize)
except AttributeError:
fd.seek(allocatesize - 1)
fd.write(b'\0')
self._allocated = True
def _create_file(self, filename, size, sparse=True):
try:
f = open(filename, "w+")
except (OSError, IOError):
raise ExecutionError("Could not open %s" % filename)
try:
if sparse:
try:
os.posix_fallocate(f.fileno(), 0, size)
except AttributeError:
# Prior to version 3.3, Python does not provide fallocate
os.ftruncate(f.fileno(), size)
else:
self.shell.log.info("Writing %d bytes" % size)
while size > 0:
write_size = min(size, 1024)
f.write("\0" * write_size)
size -= write_size
except (OSError, IOError):
os.remove(filename)
raise ExecutionError("Could not expand file to %d bytes" % size)
except OverflowError:
raise ExecutionError("The file size is too large (%d bytes)" % size)
finally:
f.close()
def mkfile(self, filename, nbytes):
"""
Create a non-sparse file of that size. Deletes and replaces existing file.
To allow efficient execution, fallocate will be tried first. This includes
``os.posix_fallocate`` on Python 3.3+ (unix) and the ``fallocate`` command
in the popular ``util-linux{,-ng}`` package.
A dd fallback will be tried too. When size < 64M, perform single-pass dd.
Otherwise do two-pass dd.
"""
if not isinstance(nbytes, int):
nbytes = int(nbytes)
if nbytes < 0:
raise ValueError(nbytes)
if os.path.isfile(filename):
os.remove(filename)
# os.posix_fallocate
if sys.version_info >= (3, 3):
# Probable errors:
# - OSError: Seen on Cygwin, libc notimpl?
# - AttributeError: What if someone runs this under...
with open(filename, 'w') as f:
try:
os.posix_fallocate(f.fileno(), 0, nbytes)
return 0
except:
# Not confident with this thing, just keep trying...
pass
# fallocate command
fn_sh = shellutil.quote((filename,))
ret = shellutil.run(u"umask 0077 && fallocate -l {0} {1}".format(nbytes, fn_sh))
if ret == 0:
return ret
logger.info("fallocate unsuccessful, falling back to dd")
# dd fallback
dd_maxbs = 64 * 1024 ** 2
dd_cmd = "umask 0077 && dd if=/dev/zero bs={0} count={1} conv=notrunc of={2}"
blocks = int(nbytes / dd_maxbs)
if blocks > 0:
ret = shellutil.run(dd_cmd.format(dd_maxbs, blocks, fn_sh)) << 8
remains = int(nbytes % dd_maxbs)
if remains > 0:
ret += shellutil.run(dd_cmd.format(remains, 1, fn_sh))
if ret == 0:
logger.info("dd successful")
else:
logger.error("dd unsuccessful")
return ret
async def download(self):
with connecting():
filename, self.size, file_type = await self.get_download_info()
if not self.output_fname:
self.output_fname = filename
if not self.output_fname:
parts = urllib.parse.urlparse(self.url)
self.output_fname = os.path.basename(parts.path)
if not self.output_fname:
tqdm.write('Unable to resolve file name from HTTP header and URL.'
'Please use the "-o" parameter.')
return 1
self.status_file = self.output_fname + '.aiodl'
if os.path.exists(self.status_file):
with open(self.status_file, 'rb') as f:
self.blocks = pickle.load(f)
downloaded_size = self.size - sum(
r.size for r in self.blocks.values()
)
# 'r+b' -- opens the file for binary random access,
# without truncates it to 0 byte (which 'w+b' does)
self.output = open(self.output_fname, 'r+b')
else:
if self.num_tasks > self.size:
tqdm.write(
'Too many tasks (%d > file size %d), using 1 task' %
(self.num_tasks, self.size)
)
self.num_tasks = 1
self.blocks = self.split()
downloaded_size = 0
self.output = open(self.output_fname, 'wb')
# pre-allocate file
try:
os.posix_fallocate(self.output.fileno(), 0, self.size)
except OSError:
pass
print_colored_kv('File', self.output_fname)
formatted_size = tqdm.format_sizeof(self.size, 'B', 1024)
if downloaded_size:
formatted_size += ' (already downloaded {})'.format(
tqdm.format_sizeof(downloaded_size, 'B', 1024))
print_colored_kv('Size', formatted_size)
print_colored_kv('Type', file_type)
tqdm.write('')
self.bar = tqdm(
initial=downloaded_size,
dynamic_ncols=True, # Suitable for window resizing
total=self.size,
unit='B', unit_scale=True, unit_divisor=1024
)
await asyncio.gather(
*(self.download_block(id) for id in self.blocks)
)
def fallocate(self):
"""
Attempts to allocate this capture slot, if it doesn't already exist.
:return: None
"""
with open(self.fn, 'wb') as file:
os.posix_fallocate(file.fileno(), 0, settings.FCAP_SIZE)
def setup_swapmem(self):
swapfile = SWAPAllocator.SWAP_FILE_PATH
with open(swapfile, 'wb') as fd:
os.posix_fallocate(
fd.fileno(), 0,
self.swap_mem_size * SWAPAllocator.MiB_TO_BYTES_FACTOR)
os.chmod(swapfile, 0o600)
run_command(f'mkswap {swapfile}; swapon {swapfile}')
def __init__(self, args=None, fd=-1, client_id=-1, nodename=None):
'''Server: args with command line stuff from command line.
Client: starts with an fd and id read from AF_UNIX socket.'''
if self._beentheredonethat:
return
self.__class__._beetnheredonethat = True
assert (self.MS_MSG_off +
self.MS_MAX_MSGLEN == self.MAILBOX_SLOTSIZE), 'Fix this NOW'
self.filesize = self.MAILBOX_MAX_SLOTS * self.MAILBOX_SLOTSIZE
if args is None:
assert fd > 0 and client_id > 0 and isinstance(nodename, str), \
'Bad call, ump!'
self.__class__.fd = fd
self.nodename = nodename
self._init_mailslot(client_id, nodename)
return
assert fd == -1 and client_id == -1, 'Cannot assign ids to server'
path = args.mailbox # Match previously written code
gr_gid = -1 # Makes no change. Try Debian, CentOS, other
for gr_name in ('libvirt-qemu', 'libvirt', 'libvirtd'):
try:
gr_gid = grp.getgrnam(gr_name).gr_gid
break
except Exception as e:
pass
if '/' not in path:
path = '/dev/shm/' + path
oldumask = os.umask(0)
try:
if not os.path.isfile(path):
fd = os.open(path, os.O_RDWR | os.O_CREAT, mode=0o666)
os.posix_fallocate(fd, 0, self.filesize)
os.fchown(fd, -1, gr_gid)
else: # Re-condition and re-use
lstat = os.lstat(path)
assert STAT.S_ISREG(lstat.st_mode), 'not a regular file'
assert lstat.st_size >= self.filesize, \
'existing size (%d) is < required (%d)' % (
lstat.st_size, self.filesize)
if lstat.st_gid != gr_gid and gr_gid > 0:
print('Changing %s to group %s' % (path, gr_name))
os.chown(path, -1, gr_gid)
if lstat.st_mode & 0o660 != 0o660: # at least
print('Changing %s to permissions 666' % path)
os.chmod(path, 0o666)
fd = os.open(path, os.O_RDWR)
except Exception as e:
raise RuntimeError('Problem with %s: %s' % (path, str(e)))
os.umask(oldumask)
self.path = path # Final absolute path
self.__class__.fd = fd
self._initialize_mailbox(args)
def test_blobcache(
nydus_anchor: NydusAnchor,
nydus_image: RafsImage,
rafs_conf: RafsConf,
compressor,
backend,
):
"""
Allocate a file with local test working directory.
Loop the file so to get a small file system which is easy to get full.
Change blob cache location the above test blobdir
"""
blobdir = "/blobdir"
blob_backend = "blob_backend"
fd = os.open(blob_backend, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
os.posix_fallocate(fd, 0, 1024 * 1024 * 4)
os.close(fd)
utils.execute(["mkfs.ext4", "-F", blob_backend])
utils.execute(["mount", blob_backend, blobdir])
rafs_conf.enable_rafs_blobcache()
rafs_conf.set_rafs_backend(backend)
rafs_conf.dump_rafs_conf()
cache_file = os.listdir(blobdir)
assert len(cache_file) == 1
rafs = RafsMount(nydus_anchor, nydus_image, rafs_conf)
rafs.mount()
assert rafs.is_mounted()
workload_gen = WorkloadGen(nydus_anchor.mount_point,
nydus_anchor.source_dir)
workload_gen.setup_workload_generator()
workload_gen.torture_read(4, 15)
nydus_anchor.start_stats_checker()
workload_gen.finish_torture_read()
nydus_anchor.stop_stats_checker()
cache_file = os.listdir(blobdir)
assert len(cache_file) >= 2
if workload_gen.io_error:
warnings.warn(
UserWarning("Rafs will return EIO if blobcache file is full"))
rafs.umount()
ret, _ = utils.execute(["umount", blobdir])
assert ret
os.unlink(blob_backend)
def mkfile(self, filename, nbytes):
"""
Create a non-sparse file of that size. Deletes and replaces existing file.
To allow efficient execution, fallocate will be tried first. This includes
``os.posix_fallocate`` on Python 3.3+ (unix) and the ``fallocate`` command
in the popular ``util-linux{,-ng}`` package.
A dd fallback will be tried too. When size < 64M, perform single-pass dd.
Otherwise do two-pass dd.
"""
if not isinstance(nbytes, int):
nbytes = int(nbytes)
if nbytes < 0:
raise ValueError(nbytes)
if os.path.isfile(filename):
os.remove(filename)
# os.posix_fallocate
if sys.version_info >= (3, 3):
# Probable errors:
# - OSError: Seen on Cygwin, libc notimpl?
# - AttributeError: What if someone runs this under...
with open(filename, 'w') as f:
try:
os.posix_fallocate(f.fileno(), 0, nbytes)
return 0
except:
# Not confident with this thing, just keep trying...
pass
# fallocate command
fn_sh = shellutil.quote((filename, ))
ret = shellutil.run(u"umask 0077 && fallocate -l {0} {1}".format(
nbytes, fn_sh))
if ret != 127: # 127 = command not found
return ret
# dd fallback
dd_maxbs = 64 * 1024**2
dd_cmd = "umask 0077 && dd if=/dev/zero bs={0} count={1} conv=notrunc of={2}"
blocks = int(nbytes / dd_maxbs)
if blocks > 0:
ret = shellutil.run(dd_cmd.format(dd_maxbs, fn_sh, blocks)) << 8
remains = int(nbytes % dd_maxbs)
if remains > 0:
ret += shellutil.run(dd_cmd.format(remains, fn_sh, 1))
return ret
def create_files(test_dir, sectorsize, numfiles):
os.chdir(test_dir)
for i in range(numfiles):
if i % 2048 == 0:
print('Created {}/{} files...'.format(i, numfiles), end='\r')
fd = os.open(str(i), os.O_WRONLY | os.O_CREAT)
try:
os.posix_fallocate(fd, 0, sectorsize)
finally:
os.close(fd)
print('Created {0}/{0} files...'.format(numfiles))
os.chdir('/')
def create_backstore(plugin, diskname, backing_device, filesize):
try:
if plugin == 1:
if os.path.isfile(backing_device):
if filesize > 17592186044416:
# max size 16TB
raise ExecutionError("The file filesize is too large (Max size:16 TB)(%d bytes)" % filesize)
elif filesize < 0:
raise ExecutionError("The file filesize should not be negative (%d bytes)" % filesize)
else:
print ("%s exists, using its size given (%s bytes)" % (backing_device, filesize))
elif os.path.exists(backing_device):
raise ExecutionError("Path %s exists but is not a file" % backing_device)
else:
if filesize > 17592186044416:
# max size 16TB
raise ExecutionError("The file filesize is too large (Max size:16 TB)(%d bytes)" % filesize)
elif filesize < 0:
raise ExecutionError("The file filesize should not be negative (%d bytes)" % filesize)
else:
try:
f = open(backing_device, "w+")
except (OSError, IOError):
raise ExecutionError("Could not open %s" % backing_device)
try:
try:
os.posix_fallocate(f.fileno(), 0, filesize)
except AttributeError:
# Prior to version 3.3, Python does not provide fallocate
os.ftruncate(f.fileno(), filesize)
except OverflowError:
raise ExecutionError("The file size is too large (%d bytes)" % size)
finally:
f.close()
fio = FileIOStorageObject(name = diskname, dev = backing_device, size = filesize)
elif plugin == 2:
bio = BlockStorageObject(name = diskname, dev = backing_device)
else :
raise ExecutionError ("set plugin is %d now, put 1 or fileio for file; 2 or block for block" % plugin)
return 0
except Exception,e:
print e
return -1
def test_different_partitions(nydus_anchor: NydusAnchor, rafs_conf):
loop_file_1 = tempfile.NamedTemporaryFile(suffix="loop")
loop_file_2 = tempfile.NamedTemporaryFile(suffix="loop")
loop_mnt_1 = tempfile.TemporaryDirectory(dir=nydus_anchor.workspace)
loop_mnt_2 = tempfile.TemporaryDirectory(dir=nydus_anchor.workspace)
os.posix_fallocate(loop_file_1.fileno(), 0, Size(400, Unit.MB).B)
os.posix_fallocate(loop_file_2.fileno(), 0, Size(400, Unit.MB).B)
utils.execute(["mkfs.ext4", "-F", loop_file_1.name])
utils.execute(["mkfs.ext4", "-F", loop_file_2.name])
utils.execute(["mount", loop_file_1.name, loop_mnt_1.name])
utils.execute(["mount", loop_file_2.name, loop_mnt_2.name])
# TODO: Put more special files into
dist1 = Distributor(loop_mnt_1.name, 5, 7)
dist1.generate_tree()
dist1.put_multiple_files(100, Size(12, Unit.KB))
dist2 = Distributor(loop_mnt_2.name, 5, 7)
dist2.generate_tree()
dist2.put_symlinks(20)
dist2.put_multiple_files(50, Size(12, Unit.KB))
Whiteout.mirror_files(dist2.files[:20], loop_mnt_2.name, loop_mnt_1.name)
parent_image = (RafsImage(nydus_anchor, loop_mnt_1.name).set_backend(
Backend.OSS).create_image())
image = RafsImage(nydus_anchor, loop_mnt_2.name)
image.set_backend(Backend.OSS).create_image(parent_image=parent_image)
rafs_conf.set_rafs_backend(Backend.OSS)
rafs = RafsMount(nydus_anchor, image, rafs_conf)
rafs.mount()
nydus_anchor.mount_overlayfs([image.rootfs(), parent_image.rootfs()])
wg = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
wg.setup_workload_generator()
wg.torture_read(5, 5)
wg.finish_torture_read()
utils.execute(["umount", loop_mnt_1.name])
utils.execute(["umount", loop_mnt_2.name])
nydus_anchor.umount_overlayfs()
def fallocate(fileobj, size):
if hasattr(os, 'posix_fallocate'):
os.posix_fallocate(fileobj.fileno(), 0, size)
else:
fileobj.truncate(size)
def mkfile(self, filename, nbytes):
"""
Create a non-sparse file of that size. Deletes and replaces existing
file.
To allow efficient execution, fallocate will be tried first. This
includes
``os.posix_fallocate`` on Python 3.3+ (unix) and the ``fallocate``
command
in the popular ``util-linux{,-ng}`` package.
A dd fallback will be tried too. When size < 64M, perform
single-pass dd.
Otherwise do two-pass dd.
"""
if not isinstance(nbytes, int):
nbytes = int(nbytes)
if nbytes <= 0:
raise ResourceDiskError("Invalid swap size [{0}]".format(nbytes))
if os.path.isfile(filename):
os.remove(filename)
# If file system is xfs, use dd right away as we have been reported that
# swap enabling fails in xfs fs when disk space is allocated with
# fallocate
ret = 0
fn_sh = shellutil.quote((filename, ))
if self.fs not in ['xfs', 'ext4']:
# os.posix_fallocate
if sys.version_info >= (3, 3):
# Probable errors:
# - OSError: Seen on Cygwin, libc notimpl?
# - AttributeError: What if someone runs this under...
fd = None
try:
fd = os.open(filename,
os.O_CREAT | os.O_WRONLY | os.O_EXCL,
stat.S_IRUSR | stat.S_IWUSR)
os.posix_fallocate(fd, 0, nbytes) # pylint: disable=no-member
return 0
except BaseException:
# Not confident with this thing, just keep trying...
pass
finally:
if fd is not None:
os.close(fd)
# fallocate command
ret = shellutil.run(u"umask 0077 && fallocate -l {0} {1}".format(
nbytes, fn_sh))
if ret == 0:
return ret
logger.info("fallocate unsuccessful, falling back to dd")
# dd fallback
dd_maxbs = 64 * 1024**2
dd_cmd = "umask 0077 && dd if=/dev/zero bs={0} count={1} " \
"conv=notrunc of={2}"
blocks = int(nbytes / dd_maxbs)
if blocks > 0:
ret = shellutil.run(dd_cmd.format(dd_maxbs, blocks, fn_sh)) << 8
remains = int(nbytes % dd_maxbs)
if remains > 0:
ret += shellutil.run(dd_cmd.format(remains, 1, fn_sh))
if ret == 0:
logger.info("dd successful")
else:
logger.error("dd unsuccessful")
return ret
def mkfile(self, filename, nbytes):
"""
Create a non-sparse file of that size. Deletes and replaces existing
file.
To allow efficient execution, fallocate will be tried first. This
includes
``os.posix_fallocate`` on Python 3.3+ (unix) and the ``fallocate``
command
in the popular ``util-linux{,-ng}`` package.
A dd fallback will be tried too. When size < 64M, perform
single-pass dd.
Otherwise do two-pass dd.
"""
if not isinstance(nbytes, int):
nbytes = int(nbytes)
if nbytes <= 0:
raise ResourceDiskError("Invalid swap size [{0}]".format(nbytes))
if os.path.isfile(filename):
os.remove(filename)
# If file system is xfs, use dd right away as we have been reported that
# swap enabling fails in xfs fs when disk space is allocated with fallocate
ret = 0
fn_sh = shellutil.quote((filename,))
if self.fs != 'xfs':
# os.posix_fallocate
if sys.version_info >= (3, 3):
# Probable errors:
# - OSError: Seen on Cygwin, libc notimpl?
# - AttributeError: What if someone runs this under...
fd = None
try:
fd = os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_EXCL, stat.S_IRUSR | stat.S_IWUSR)
os.posix_fallocate(fd, 0, nbytes)
return 0
except:
# Not confident with this thing, just keep trying...
pass
finally:
if fd is not None:
os.close(fd)
# fallocate command
ret = shellutil.run(
u"umask 0077 && fallocate -l {0} {1}".format(nbytes, fn_sh))
if ret == 0:
return ret
logger.info("fallocate unsuccessful, falling back to dd")
# dd fallback
dd_maxbs = 64 * 1024 ** 2
dd_cmd = "umask 0077 && dd if=/dev/zero bs={0} count={1} " \
"conv=notrunc of={2}"
blocks = int(nbytes / dd_maxbs)
if blocks > 0:
ret = shellutil.run(dd_cmd.format(dd_maxbs, blocks, fn_sh)) << 8
remains = int(nbytes % dd_maxbs)
if remains > 0:
ret += shellutil.run(dd_cmd.format(remains, 1, fn_sh))
if ret == 0:
logger.info("dd successful")
else:
logger.error("dd unsuccessful")
return ret