Exemple #1
0
 def _create_file(self, filename, size, sparse=True):
     try:
         f = open(filename, "w+")
     except (OSError, IOError):
         raise ExecutionError("Could not open %s" % filename)
     try:
         if sparse:
             try:
                 os.posix_fallocate(f.fileno(), 0, size)
             except AttributeError:
                 # Prior to version 3.3, Python does not provide fallocate
                 os.ftruncate(f.fileno(), size)
         else:
             self.shell.log.info("Writing %d bytes" % size)
             while size > 0:
                 write_size = min(size, 1024)
                 f.write("\0" * write_size)
                 size -= write_size
     except (OSError, IOError):
         os.remove(filename)
         raise ExecutionError("Could not expand file to %d bytes" % size)
     except OverflowError:
         raise ExecutionError("The file size is too large (%d bytes)" %
                              size)
     finally:
         f.close()
Exemple #2
0
 def _allocate_disk_space(self):
     # type: (Descriptor) -> None
     """Perform file allocation (possibly sparse)
     :param Descriptor self: this
     """
     with self._meta_lock:
         if self._allocated or self._offset != 0:
             return
         # set local path view
         allocatesize = self._set_final_path_view()
         # check if path already exists and is of sufficient size
         if (not self.final_path.exists() or
                 self.final_path.stat().st_size != allocatesize):
             # create parent path
             self.final_path.parent.mkdir(
                 mode=0o750, parents=True, exist_ok=True)
             # allocate file
             with self.final_path.open('wb') as fd:
                 if allocatesize > 0:
                     try:
                         os.posix_fallocate(fd.fileno(), 0, allocatesize)
                     except AttributeError:
                         fd.seek(allocatesize - 1)
                         fd.write(b'\0')
         self._allocated = True
 def _create_file(self, filename, size, sparse=True):
     try:
         f = open(filename, "w+")
     except (OSError, IOError):
         raise ExecutionError("Could not open %s" % filename)
     try:
         if sparse:
             try:
                 os.posix_fallocate(f.fileno(), 0, size)
             except AttributeError:
                 # Prior to version 3.3, Python does not provide fallocate
                 os.ftruncate(f.fileno(), size)
         else:
             self.shell.log.info("Writing %d bytes" % size)
             while size > 0:
                 write_size = min(size, 1024)
                 f.write("\0" * write_size)
                 size -= write_size
     except (OSError, IOError):
         os.remove(filename)
         raise ExecutionError("Could not expand file to %d bytes" % size)
     except OverflowError:
         raise ExecutionError("The file size is too large (%d bytes)" % size)
     finally:
         f.close()
Exemple #4
0
    def mkfile(self, filename, nbytes):
        """
        Create a non-sparse file of that size. Deletes and replaces existing file.

        To allow efficient execution, fallocate will be tried first. This includes
        ``os.posix_fallocate`` on Python 3.3+ (unix) and the ``fallocate`` command
        in the popular ``util-linux{,-ng}`` package.

        A dd fallback will be tried too. When size < 64M, perform single-pass dd.
        Otherwise do two-pass dd.
        """

        if not isinstance(nbytes, int):
            nbytes = int(nbytes)

        if nbytes < 0:
            raise ValueError(nbytes)

        if os.path.isfile(filename):
            os.remove(filename)

        # os.posix_fallocate
        if sys.version_info >= (3, 3):
            # Probable errors:
            #  - OSError: Seen on Cygwin, libc notimpl?
            #  - AttributeError: What if someone runs this under...
            with open(filename, 'w') as f:
                try:
                    os.posix_fallocate(f.fileno(), 0, nbytes)
                    return 0
                except:
                    # Not confident with this thing, just keep trying...
                    pass

        # fallocate command
        fn_sh = shellutil.quote((filename,))
        ret = shellutil.run(u"umask 0077 && fallocate -l {0} {1}".format(nbytes, fn_sh))
        if ret == 0:
            return ret

        logger.info("fallocate unsuccessful, falling back to dd")

        # dd fallback
        dd_maxbs = 64 * 1024 ** 2
        dd_cmd = "umask 0077 && dd if=/dev/zero bs={0} count={1} conv=notrunc of={2}"

        blocks = int(nbytes / dd_maxbs)
        if blocks > 0:
            ret = shellutil.run(dd_cmd.format(dd_maxbs, blocks, fn_sh)) << 8

        remains = int(nbytes % dd_maxbs)
        if remains > 0:
            ret += shellutil.run(dd_cmd.format(remains, 1, fn_sh))

        if ret == 0:
            logger.info("dd successful")
        else:
            logger.error("dd unsuccessful")

        return ret
Exemple #5
0
    async def download(self):
        with connecting():
            filename, self.size, file_type = await self.get_download_info()

        if not self.output_fname:
            self.output_fname = filename
        if not self.output_fname:
            parts = urllib.parse.urlparse(self.url)
            self.output_fname = os.path.basename(parts.path)
        if not self.output_fname:
            tqdm.write('Unable to resolve file name from HTTP header and URL.'
                       'Please use the "-o" parameter.')
            return 1
        self.status_file = self.output_fname + '.aiodl'

        if os.path.exists(self.status_file):
            with open(self.status_file, 'rb') as f:
                self.blocks = pickle.load(f)
            downloaded_size = self.size - sum(
                r.size for r in self.blocks.values()
            )
            # 'r+b' -- opens the file for binary random access,
            # without truncates it to 0 byte (which 'w+b' does)
            self.output = open(self.output_fname, 'r+b')
        else:
            if self.num_tasks > self.size:
                tqdm.write(
                    'Too many tasks (%d > file size %d), using 1 task' %
                    (self.num_tasks, self.size)
                )
                self.num_tasks = 1
            self.blocks = self.split()
            downloaded_size = 0
            self.output = open(self.output_fname, 'wb')
            # pre-allocate file
            try:
                os.posix_fallocate(self.output.fileno(), 0, self.size)
            except OSError:
                pass

        print_colored_kv('File', self.output_fname)
        formatted_size = tqdm.format_sizeof(self.size, 'B', 1024)
        if downloaded_size:
            formatted_size += ' (already downloaded {})'.format(
                tqdm.format_sizeof(downloaded_size, 'B', 1024))
        print_colored_kv('Size', formatted_size)
        print_colored_kv('Type', file_type)
        tqdm.write('')

        self.bar = tqdm(
            initial=downloaded_size,
            dynamic_ncols=True,  # Suitable for window resizing
            total=self.size,
            unit='B', unit_scale=True, unit_divisor=1024
        )

        await asyncio.gather(
            *(self.download_block(id) for id in self.blocks)
        )
Exemple #6
0
    def fallocate(self):
        """
        Attempts to allocate this capture slot, if it doesn't already exist.
        :return: None
        """

        with open(self.fn, 'wb') as file:
            os.posix_fallocate(file.fileno(), 0, settings.FCAP_SIZE)
Exemple #7
0
 def setup_swapmem(self):
     swapfile = SWAPAllocator.SWAP_FILE_PATH
     with open(swapfile, 'wb') as fd:
         os.posix_fallocate(
             fd.fileno(), 0,
             self.swap_mem_size * SWAPAllocator.MiB_TO_BYTES_FACTOR)
     os.chmod(swapfile, 0o600)
     run_command(f'mkswap {swapfile}; swapon {swapfile}')
Exemple #8
0
    def __init__(self, args=None, fd=-1, client_id=-1, nodename=None):
        '''Server: args with command line stuff from command line.
           Client: starts with an fd and id read from AF_UNIX socket.'''
        if self._beentheredonethat:
            return
        self.__class__._beetnheredonethat = True

        assert (self.MS_MSG_off +
                self.MS_MAX_MSGLEN == self.MAILBOX_SLOTSIZE), 'Fix this NOW'
        self.filesize = self.MAILBOX_MAX_SLOTS * self.MAILBOX_SLOTSIZE

        if args is None:
            assert fd > 0 and client_id > 0 and isinstance(nodename, str), \
                'Bad call, ump!'
            self.__class__.fd = fd
            self.nodename = nodename
            self._init_mailslot(client_id, nodename)
            return
        assert fd == -1 and client_id == -1, 'Cannot assign ids to server'

        path = args.mailbox  # Match previously written code
        gr_gid = -1  # Makes no change.  Try Debian, CentOS, other
        for gr_name in ('libvirt-qemu', 'libvirt', 'libvirtd'):
            try:
                gr_gid = grp.getgrnam(gr_name).gr_gid
                break
            except Exception as e:
                pass

        if '/' not in path:
            path = '/dev/shm/' + path
        oldumask = os.umask(0)
        try:
            if not os.path.isfile(path):
                fd = os.open(path, os.O_RDWR | os.O_CREAT, mode=0o666)
                os.posix_fallocate(fd, 0, self.filesize)
                os.fchown(fd, -1, gr_gid)
            else:  # Re-condition and re-use
                lstat = os.lstat(path)
                assert STAT.S_ISREG(lstat.st_mode), 'not a regular file'
                assert lstat.st_size >= self.filesize, \
                    'existing size (%d) is < required (%d)' % (
                        lstat.st_size, self.filesize)
                if lstat.st_gid != gr_gid and gr_gid > 0:
                    print('Changing %s to group %s' % (path, gr_name))
                    os.chown(path, -1, gr_gid)
                if lstat.st_mode & 0o660 != 0o660:  # at least
                    print('Changing %s to permissions 666' % path)
                    os.chmod(path, 0o666)
                fd = os.open(path, os.O_RDWR)
        except Exception as e:
            raise RuntimeError('Problem with %s: %s' % (path, str(e)))

        os.umask(oldumask)

        self.path = path  # Final absolute path
        self.__class__.fd = fd
        self._initialize_mailbox(args)
Exemple #9
0
def test_blobcache(
    nydus_anchor: NydusAnchor,
    nydus_image: RafsImage,
    rafs_conf: RafsConf,
    compressor,
    backend,
):
    """
    Allocate a file with local test working directory.
    Loop the file so to get a small file system which is easy to get full.
    Change blob cache location the above test blobdir
    """

    blobdir = "/blobdir"

    blob_backend = "blob_backend"
    fd = os.open(blob_backend, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
    os.posix_fallocate(fd, 0, 1024 * 1024 * 4)
    os.close(fd)

    utils.execute(["mkfs.ext4", "-F", blob_backend])
    utils.execute(["mount", blob_backend, blobdir])

    rafs_conf.enable_rafs_blobcache()
    rafs_conf.set_rafs_backend(backend)
    rafs_conf.dump_rafs_conf()

    cache_file = os.listdir(blobdir)
    assert len(cache_file) == 1

    rafs = RafsMount(nydus_anchor, nydus_image, rafs_conf)
    rafs.mount()
    assert rafs.is_mounted()

    workload_gen = WorkloadGen(nydus_anchor.mount_point,
                               nydus_anchor.source_dir)

    workload_gen.setup_workload_generator()
    workload_gen.torture_read(4, 15)

    nydus_anchor.start_stats_checker()
    workload_gen.finish_torture_read()
    nydus_anchor.stop_stats_checker()

    cache_file = os.listdir(blobdir)
    assert len(cache_file) >= 2

    if workload_gen.io_error:
        warnings.warn(
            UserWarning("Rafs will return EIO if blobcache file is full"))

    rafs.umount()

    ret, _ = utils.execute(["umount", blobdir])
    assert ret

    os.unlink(blob_backend)
Exemple #10
0
    def mkfile(self, filename, nbytes):
        """
        Create a non-sparse file of that size. Deletes and replaces existing file.

        To allow efficient execution, fallocate will be tried first. This includes
        ``os.posix_fallocate`` on Python 3.3+ (unix) and the ``fallocate`` command
        in the popular ``util-linux{,-ng}`` package.

        A dd fallback will be tried too. When size < 64M, perform single-pass dd.
        Otherwise do two-pass dd.
        """

        if not isinstance(nbytes, int):
            nbytes = int(nbytes)

        if nbytes < 0:
            raise ValueError(nbytes)

        if os.path.isfile(filename):
            os.remove(filename)

        # os.posix_fallocate
        if sys.version_info >= (3, 3):
            # Probable errors:
            #  - OSError: Seen on Cygwin, libc notimpl?
            #  - AttributeError: What if someone runs this under...
            with open(filename, 'w') as f:
                try:
                    os.posix_fallocate(f.fileno(), 0, nbytes)
                    return 0
                except:
                    # Not confident with this thing, just keep trying...
                    pass

        # fallocate command
        fn_sh = shellutil.quote((filename, ))
        ret = shellutil.run(u"umask 0077 && fallocate -l {0} {1}".format(
            nbytes, fn_sh))
        if ret != 127:  # 127 = command not found
            return ret

        # dd fallback
        dd_maxbs = 64 * 1024**2
        dd_cmd = "umask 0077 && dd if=/dev/zero bs={0} count={1} conv=notrunc of={2}"

        blocks = int(nbytes / dd_maxbs)
        if blocks > 0:
            ret = shellutil.run(dd_cmd.format(dd_maxbs, fn_sh, blocks)) << 8

        remains = int(nbytes % dd_maxbs)
        if remains > 0:
            ret += shellutil.run(dd_cmd.format(remains, fn_sh, 1))

        return ret
Exemple #11
0
def create_files(test_dir, sectorsize, numfiles):
    os.chdir(test_dir)
    for i in range(numfiles):
        if i % 2048 == 0:
            print('Created {}/{} files...'.format(i, numfiles), end='\r')
        fd = os.open(str(i), os.O_WRONLY | os.O_CREAT)
        try:
            os.posix_fallocate(fd, 0, sectorsize)
        finally:
            os.close(fd)
    print('Created {0}/{0} files...'.format(numfiles))
    os.chdir('/')
def create_backstore(plugin, diskname, backing_device, filesize):
	try:

		if plugin == 1:
			if os.path.isfile(backing_device):
				if filesize > 17592186044416:
				# max size 16TB
					raise ExecutionError("The file filesize is too large (Max size:16 TB)(%d bytes)" % filesize)
				elif filesize < 0:
					raise ExecutionError("The file filesize should not be negative (%d bytes)" % filesize)
				else:
					print ("%s exists, using its size given (%s bytes)" % (backing_device, filesize))
			elif os.path.exists(backing_device):
					raise ExecutionError("Path %s exists but is not a file" % backing_device)
			else:
				if filesize > 17592186044416:
				# max size 16TB
					raise ExecutionError("The file filesize is too large (Max size:16 TB)(%d bytes)" % filesize)
				elif filesize < 0:
					raise ExecutionError("The file filesize should not be negative (%d bytes)" % filesize)
				else:
					try:
						f = open(backing_device, "w+")
					except (OSError, IOError):
						raise ExecutionError("Could not open %s" % backing_device)
					try:
						try:
							os.posix_fallocate(f.fileno(), 0, filesize)

						except AttributeError:
		                    # Prior to version 3.3, Python does not provide fallocate
							os.ftruncate(f.fileno(), filesize)
					except OverflowError:
						raise ExecutionError("The file size is too large (%d bytes)" % size)

					finally:
						f.close()

			fio = FileIOStorageObject(name = diskname, dev = backing_device, size = filesize)
		elif plugin == 2:
			bio = BlockStorageObject(name = diskname, dev = backing_device)
		else :
			raise ExecutionError ("set plugin is %d now, put 1 or fileio for file; 2 or block for block" % plugin)
		
		return 0

	except Exception,e:

		print e

		return  -1
Exemple #13
0
def test_different_partitions(nydus_anchor: NydusAnchor, rafs_conf):
    loop_file_1 = tempfile.NamedTemporaryFile(suffix="loop")
    loop_file_2 = tempfile.NamedTemporaryFile(suffix="loop")
    loop_mnt_1 = tempfile.TemporaryDirectory(dir=nydus_anchor.workspace)
    loop_mnt_2 = tempfile.TemporaryDirectory(dir=nydus_anchor.workspace)

    os.posix_fallocate(loop_file_1.fileno(), 0, Size(400, Unit.MB).B)
    os.posix_fallocate(loop_file_2.fileno(), 0, Size(400, Unit.MB).B)

    utils.execute(["mkfs.ext4", "-F", loop_file_1.name])
    utils.execute(["mkfs.ext4", "-F", loop_file_2.name])
    utils.execute(["mount", loop_file_1.name, loop_mnt_1.name])
    utils.execute(["mount", loop_file_2.name, loop_mnt_2.name])

    # TODO: Put more special files into
    dist1 = Distributor(loop_mnt_1.name, 5, 7)
    dist1.generate_tree()
    dist1.put_multiple_files(100, Size(12, Unit.KB))

    dist2 = Distributor(loop_mnt_2.name, 5, 7)
    dist2.generate_tree()
    dist2.put_symlinks(20)
    dist2.put_multiple_files(50, Size(12, Unit.KB))

    Whiteout.mirror_files(dist2.files[:20], loop_mnt_2.name, loop_mnt_1.name)

    parent_image = (RafsImage(nydus_anchor, loop_mnt_1.name).set_backend(
        Backend.OSS).create_image())

    image = RafsImage(nydus_anchor, loop_mnt_2.name)
    image.set_backend(Backend.OSS).create_image(parent_image=parent_image)

    rafs_conf.set_rafs_backend(Backend.OSS)
    rafs = RafsMount(nydus_anchor, image, rafs_conf)
    rafs.mount()

    nydus_anchor.mount_overlayfs([image.rootfs(), parent_image.rootfs()])

    wg = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
    wg.setup_workload_generator()
    wg.torture_read(5, 5)
    wg.finish_torture_read()

    utils.execute(["umount", loop_mnt_1.name])
    utils.execute(["umount", loop_mnt_2.name])

    nydus_anchor.umount_overlayfs()
Exemple #14
0
def fallocate(fileobj, size):
    if hasattr(os, 'posix_fallocate'):
        os.posix_fallocate(fileobj.fileno(), 0, size)
    else:
        fileobj.truncate(size)
Exemple #15
0
    def mkfile(self, filename, nbytes):
        """
        Create a non-sparse file of that size. Deletes and replaces existing
        file.

        To allow efficient execution, fallocate will be tried first. This
        includes
        ``os.posix_fallocate`` on Python 3.3+ (unix) and the ``fallocate``
        command
        in the popular ``util-linux{,-ng}`` package.

        A dd fallback will be tried too. When size < 64M, perform
        single-pass dd.
        Otherwise do two-pass dd.
        """

        if not isinstance(nbytes, int):
            nbytes = int(nbytes)

        if nbytes <= 0:
            raise ResourceDiskError("Invalid swap size [{0}]".format(nbytes))

        if os.path.isfile(filename):
            os.remove(filename)

        # If file system is xfs, use dd right away as we have been reported that
        # swap enabling fails in xfs fs when disk space is allocated with
        # fallocate
        ret = 0
        fn_sh = shellutil.quote((filename, ))
        if self.fs not in ['xfs', 'ext4']:
            # os.posix_fallocate
            if sys.version_info >= (3, 3):
                # Probable errors:
                #  - OSError: Seen on Cygwin, libc notimpl?
                #  - AttributeError: What if someone runs this under...
                fd = None

                try:
                    fd = os.open(filename,
                                 os.O_CREAT | os.O_WRONLY | os.O_EXCL,
                                 stat.S_IRUSR | stat.S_IWUSR)
                    os.posix_fallocate(fd, 0, nbytes)  # pylint: disable=no-member
                    return 0
                except BaseException:
                    # Not confident with this thing, just keep trying...
                    pass
                finally:
                    if fd is not None:
                        os.close(fd)

            # fallocate command
            ret = shellutil.run(u"umask 0077 && fallocate -l {0} {1}".format(
                nbytes, fn_sh))
            if ret == 0:
                return ret

            logger.info("fallocate unsuccessful, falling back to dd")

        # dd fallback
        dd_maxbs = 64 * 1024**2
        dd_cmd = "umask 0077 && dd if=/dev/zero bs={0} count={1} " \
                 "conv=notrunc of={2}"

        blocks = int(nbytes / dd_maxbs)
        if blocks > 0:
            ret = shellutil.run(dd_cmd.format(dd_maxbs, blocks, fn_sh)) << 8

        remains = int(nbytes % dd_maxbs)
        if remains > 0:
            ret += shellutil.run(dd_cmd.format(remains, 1, fn_sh))

        if ret == 0:
            logger.info("dd successful")
        else:
            logger.error("dd unsuccessful")

        return ret
Exemple #16
0
    def mkfile(self, filename, nbytes):
        """
        Create a non-sparse file of that size. Deletes and replaces existing
        file.

        To allow efficient execution, fallocate will be tried first. This
        includes
        ``os.posix_fallocate`` on Python 3.3+ (unix) and the ``fallocate``
        command
        in the popular ``util-linux{,-ng}`` package.

        A dd fallback will be tried too. When size < 64M, perform
        single-pass dd.
        Otherwise do two-pass dd.
        """

        if not isinstance(nbytes, int):
            nbytes = int(nbytes)

        if nbytes <= 0:
            raise ResourceDiskError("Invalid swap size [{0}]".format(nbytes))

        if os.path.isfile(filename):
            os.remove(filename)

        # If file system is xfs, use dd right away as we have been reported that
        # swap enabling fails in xfs fs when disk space is allocated with fallocate
        ret = 0
        fn_sh = shellutil.quote((filename,))
        if self.fs != 'xfs':
            # os.posix_fallocate
            if sys.version_info >= (3, 3):
                # Probable errors:
                #  - OSError: Seen on Cygwin, libc notimpl?
                #  - AttributeError: What if someone runs this under...
                fd = None

                try:
                    fd = os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_EXCL, stat.S_IRUSR | stat.S_IWUSR)
                    os.posix_fallocate(fd, 0, nbytes)
                    return 0
                except:
                    # Not confident with this thing, just keep trying...
                    pass
                finally:
                    if fd is not None:
                        os.close(fd)

            # fallocate command
            ret = shellutil.run(
                u"umask 0077 && fallocate -l {0} {1}".format(nbytes, fn_sh))
            if ret == 0:
                return ret

            logger.info("fallocate unsuccessful, falling back to dd")

        # dd fallback
        dd_maxbs = 64 * 1024 ** 2
        dd_cmd = "umask 0077 && dd if=/dev/zero bs={0} count={1} " \
                 "conv=notrunc of={2}"

        blocks = int(nbytes / dd_maxbs)
        if blocks > 0:
            ret = shellutil.run(dd_cmd.format(dd_maxbs, blocks, fn_sh)) << 8

        remains = int(nbytes % dd_maxbs)
        if remains > 0:
            ret += shellutil.run(dd_cmd.format(remains, 1, fn_sh))

        if ret == 0:
            logger.info("dd successful")
        else:
            logger.error("dd unsuccessful")

        return ret