Example #1
0
    def setUp(self):
        """
        Setup scripts/disks to memory hog
        """
        smm = SoftwareManager()
        self.memsize = self.params.get('memory_size', default=None)
        if not self.memsize:
            self.memsize = int(memory.meminfo.MemFree.b * 0.9)
        self.file_type = self.params.get('file_type', default=None)
        deps = ['gcc']
        detected_distro = distro.detect()
        if detected_distro.name in ["Ubuntu", "debian"]:
            deps.extend(['libnuma-dev'])
        elif detected_distro.name in ["centos", "rhel", "fedora"]:
            deps.extend(['numactl-devel'])
        else:
            deps.extend(['libnuma-devel'])
        if self.file_type:
            if self.file_type == 'nvdimm':
                deps.extend(['ndctl'])
                if detected_distro.name == "rhel":
                    deps.extend(['daxctl'])
        for package in deps:
            if not smm.check_installed(package) and not smm.install(package):
                self.cancel('%s is needed for the test to be run' % package)

        srcdir = os.path.join(self.workdir, 'memhog')
        if not os.path.exists(srcdir):
            os.makedirs(srcdir)
        for fname in ['memhog.c', 'util.c', 'util.h']:
            fetch_file = self.fetch_asset(
                fname,
                locations=[
                    'https://raw.githubusercontent.com/numactl/'
                    'numactl/master/%s' % fname
                ],
                expire='7d')
            shutil.copyfile(fetch_file, os.path.join(srcdir, fname))

        if self.file_type:
            if self.file_type == 'nvdimm':
                self.back_file = self.setup_nvdimm()
                disk_size = int(disk.freespace(self.mnt_dir) * 0.95)
            else:
                self.back_file = '/tmp/back_file'
                disk_size = int(disk.freespace('/tmp') * 0.95)
            if self.memsize > disk_size:
                self.memsize = disk_size
            cmd = 'fallocate -l %s %s' % (self.memsize, self.back_file)
            if process.system(cmd, ignore_status=True):
                self.cancel('Could not create file to be backed')

        os.chdir(srcdir)
        if process.system('gcc -lnuma -o memhog memhog.c util.c'):
            self.cancel('Compiling source failed')
Example #2
0
    def setUp(self):
        """
        Downloads a copy of the linux kernel, calculate an estimated size of
        the uncompressed tarball, use this value to calculate the number of
        copies of the linux kernel that will be uncompressed.
        """
        self.nfail = 0
        base_dir = self.teststmpdir
        if disk.freespace("/home") > disk.freespace(self.teststmpdir):
            base_dir = "/home"
            self.log.info("Using %s as base dir for the test" % base_dir)
        self.tmpdir = tempfile.mkdtemp(dir=base_dir)
        self.tmpdir = self.params.get('dir_to_extract', default=self.tmpdir)
        self.base_dir = os.path.join(self.tmpdir, 'linux.orig')
        tarball_base = self.params.get('tarball_base',
                                       default='linux-2.6.18.tar.bz2')
        kernel_repo = self.params.get('kernel_repo',
                                      default='http://www.kernel.org/pub/'
                                      'linux/kernel/v2.6')
        tarball_url = os.path.join(kernel_repo, tarball_base)
        tarball_md5 = self.params.get('tarball_md5',
                                      default='296a6d150d260144639c3664d127d1'
                                      '74')
        parallel = self.params.get('parallel', default=True)
        self.parallel = parallel
        self.log.info('Downloading linux kernel tarball')
        self.tarball = self.fetch_asset(tarball_url,
                                        asset_hash=tarball_md5,
                                        algorithm='md5')
        size_tarball = os.path.getsize(self.tarball) // 1024 // 1024

        # Estimation of the tarball size after uncompression
        os.chdir(self.tmpdir)
        # This is the reference copy of the linux tarball
        # that will be used for subsequent comparisons
        self.log.info('Unpacking base copy')
        archive.extract(self.tarball, self.base_dir)
        est_size = int(
            process.system_output(
                'du -sb %s' % self.base_dir).split()[0].decode()) // 1048576

        self.sim_cps = self.get_sim_cps(est_size)
        self.log.info('Source file: %s', tarball_base)
        self.log.info('Megabytes per copy: %s', size_tarball)
        self.log.info('Estimated size after uncompression: %s', est_size)
        self.log.info('Number of copies: %s', self.sim_cps)
        self.log.info('Parallel: %s', parallel)

        # Verify if space is available in disk
        disk_free_mb = (disk.freespace(self.tmpdir) // 1024) // 1024
        if disk_free_mb < (est_size * self.sim_cps):
            self.cancel("Space not available to extract the %s linux tars\n"
                        "Mount and Use other partitions in dir_to_extract arg "
                        "to run the test" % self.sim_cps)
Example #3
0
 def setUp(self):
     self.iter = int(self.params.get('iterations', default='5'))
     self.memsize = int(self.params.get(
         'mem_size', default=memory.memtotal() * 0.9))
     self.ddfile = os.path.join(self.workdir, 'ddfile')
     if (disk.freespace(self.workdir) / 1024) < self.memsize:
         self.cancel('%sM is needed for the test to be run' % self.memsize)
Example #4
0
    def setUp(self):
        '''
        Build interbench
        Source:
        http://ck.kolivas.org/apps/interbench/interbench-0.31.tar.bz2
        '''
        sm_manager = SoftwareManager()
        for pkg in ['gcc', 'patch']:
            if (not sm_manager.check_installed(pkg)
                    and not sm_manager.install(pkg)):
                self.cancel("%s is needed for the test to be run" % pkg)

        disk_free_mb = (disk.freespace(self.teststmpdir) / 1024) / 1024
        if memory.memtotal() / 1024 > disk_free_mb:
            self.cancel('Disk space is less than total memory. Skipping test')

        tarball = self.fetch_asset('http://slackware.cs.utah.edu/pub/kernel'
                                   '.org/pub/linux/kernel/people/ck/apps/'
                                   'interbench/interbench-0.31.tar.gz')
        data_dir = os.path.abspath(self.datadir)
        archive.extract(tarball, self.srcdir)
        version = os.path.basename(tarball.split('.tar.')[0])
        self.srcdir = os.path.join(self.srcdir, version)

        # Patch for make file
        os.chdir(self.srcdir)
        makefile_patch = 'patch -p1 < %s ' % (os.path.join(
            data_dir, 'makefile_fix.patch'))
        process.run(makefile_patch, shell=True)

        build.make(self.srcdir)
    def setUp(self):
        '''
        Build interbench
        Source:
        http://ck.kolivas.org/apps/interbench/interbench-0.31.tar.bz2
        '''
        sm_manager = SoftwareManager()
        for pkg in ['gcc', 'patch']:
            if (not sm_manager.check_installed(pkg) and not
                    sm_manager.install(pkg)):
                self.cancel("%s is needed for the test to be run" % pkg)

        disk_free_mb = (disk.freespace(self.teststmpdir) / 1024) / 1024
        if memory.memtotal()/1024 > disk_free_mb:
            self.cancel('Disk space is less than total memory. Skipping test')

        tarball = self.fetch_asset('http://slackware.cs.utah.edu/pub/kernel'
                                   '.org/pub/linux/kernel/people/ck/apps/'
                                   'interbench/interbench-0.31.tar.gz')
        data_dir = os.path.abspath(self.datadir)
        archive.extract(tarball, self.srcdir)
        version = os.path.basename(tarball.split('.tar.')[0])
        self.sourcedir = os.path.join(self.srcdir, version)

        # Patch for make file
        os.chdir(self.sourcedir)
        makefile_patch = 'patch -p1 < %s ' % (
            os.path.join(data_dir, 'makefile_fix.patch'))
        process.run(makefile_patch, shell=True)

        build.make(self.sourcedir)
Example #6
0
    def setUp(self):
        '''
        Build interbench
        Source:
        http://ck.kolivas.org/apps/interbench/interbench-0.31.tar.bz2
        '''
        sm_manager = SoftwareManager()
        for pkg in ['gcc', 'patch']:
            if (not sm_manager.check_installed(pkg)
                    and not sm_manager.install(pkg)):
                self.cancel("%s is needed for the test to be run" % pkg)

        disk_free_b = disk.freespace(self.teststmpdir)
        if memory.meminfo.MemTotal.b > disk_free_b:
            self.cancel('Disk space is less than total memory. Skipping test')

        tarball = self.fetch_asset(
            'http://ck.kolivas.org/apps/interbench/interbench-0.31.tar.bz2')
        archive.extract(tarball, self.workdir)
        version = os.path.basename(tarball.split('.tar.')[0])
        self.sourcedir = os.path.join(self.workdir, version)

        # Patch for make file
        os.chdir(self.sourcedir)
        makefile_patch = 'patch -p1 < %s ' % self.get_data(
            'makefile_fix.patch')
        process.run(makefile_patch, shell=True)

        build.make(self.sourcedir)
Example #7
0
    def setUp(self):
        self.iter = int(self.params.get('iterations', default='5'))
        dir_to_use = self.params.get('dir_to_use', default=None)
        self.memsize = int(self.params.get(
            'mem_size', default=memory.meminfo.MemFree.k * 0.9))
        if not dir_to_use:
            base_dir = self.teststmpdir
            if disk.freespace("/home") > disk.freespace(self.teststmpdir):
                base_dir = "/home"
        else:
            base_dir = dir_to_use

        self.log.info("Using %s as base dir for the test" % base_dir)
        self.tmpdir = tempfile.mkdtemp(dir=base_dir)
        self.ddfile = os.path.join(self.tmpdir, 'ddfile')
        if (disk.freespace(self.tmpdir) // 1024) < self.memsize:
            self.cancel('%sM is needed for the test to be run' %
                        (self.memsize // 1024))
Example #8
0
    def test(self):
        """
        Execute 'stress' with proper arguments.
        """
        length = self.params.get('stress_lenght', default=60)
        threads = self.params.get('threads', default=None)
        memory_per_thread = self.params.get('memory_per_thread', default=None)
        file_size_per_thread = self.params.get('file_size_per_thread',
                                               default=None)
        if threads is None:
            # We will use 2 workers of each type for each CPU detected
            threads = 2 * multiprocessing.cpu_count()

        if memory_per_thread is None:
            # Sometimes the default memory used by each memory worker (256 M)
            # might make our machine go OOM and then funny things might start
            # to  happen. Let's avoid that.
            mb = (memory.freememtotal() +
                  memory.read_from_meminfo('SwapFree') / 2)
            memory_per_thread = (mb * 1024) / threads

        if file_size_per_thread is None:
            # Even though unlikely, it's good to prevent from allocating more
            # disk than this machine actually has on its autotest directory
            # (limit the amount of disk used to max of 90 % of free space)
            free_disk = disk.freespace(self.sourcedir)
            file_size_per_thread = 1024**2
            if (0.9 * free_disk) < file_size_per_thread * threads:
                file_size_per_thread = (0.9 * free_disk) / threads

        # Number of CPU workers spinning on sqrt()
        args = '--cpu %d ' % threads
        # Number of IO workers spinning on sync()
        args += '--io %d ' % threads
        # Number of Memory workers spinning on malloc()/free()
        args += '--vm %d ' % threads
        # Amount of memory used per each worker
        args += '--vm-bytes %d ' % memory_per_thread
        # Number of HD workers spinning on write()/ulink()
        args += '--hdd %d ' % threads
        # Size of the files created by each worker in bytes
        args += '--hdd-bytes %d ' % file_size_per_thread
        # Time for which the stress test will run
        args += '--timeout %d ' % length
        # Verbose flag
        args += '--verbose'

        os.chdir(self.sourcedir)
        cmd = ('./src/stress %s' % args)
        process.run(cmd)
Example #9
0
    def test(self):
        """
        Execute 'stress' with proper arguments.
        """
        length = self.params.get('stress_lenght', default=60)
        threads = self.params.get('threads', default=None)
        memory_per_thread = self.params.get('memory_per_thread', default=None)
        file_size_per_thread = self.params.get('file_size_per_thread',
                                               default=None)
        if threads is None:
            # We will use 2 workers of each type for each CPU detected
            threads = 2 * multiprocessing.cpu_count()

        if memory_per_thread is None:
            # Sometimes the default memory used by each memory worker (256 M)
            # might make our machine go OOM and then funny things might start
            # to  happen. Let's avoid that.
            mb = (memory.freememtotal() +
                  memory.read_from_meminfo('SwapFree') / 2)
            memory_per_thread = (mb * 1024) / threads

        if file_size_per_thread is None:
            # Even though unlikely, it's good to prevent from allocating more
            # disk than this machine actually has on its autotest directory
            # (limit the amount of disk used to max of 90 % of free space)
            free_disk = disk.freespace(self.sourcedir)
            file_size_per_thread = 1024 ** 2
            if (0.9 * free_disk) < file_size_per_thread * threads:
                file_size_per_thread = (0.9 * free_disk) / threads

        # Number of CPU workers spinning on sqrt()
        args = '--cpu %d ' % threads
        # Number of IO workers spinning on sync()
        args += '--io %d ' % threads
        # Number of Memory workers spinning on malloc()/free()
        args += '--vm %d ' % threads
        # Amount of memory used per each worker
        args += '--vm-bytes %d ' % memory_per_thread
        # Number of HD workers spinning on write()/ulink()
        args += '--hdd %d ' % threads
        # Size of the files created by each worker in bytes
        args += '--hdd-bytes %d ' % file_size_per_thread
        # Time for which the stress test will run
        args += '--timeout %d ' % length
        # Verbose flag
        args += '--verbose'

        os.chdir(self.sourcedir)
        cmd = ('./src/stress %s' % args)
        process.run(cmd)
    def setUp(self):
        '''
        Sets required params for dd workload and mounts the tmpfs
        '''

        # Get required mem info
        self.mem_path = os.path.join(data_dir.get_tmp_dir(), 'thp_space')
        self.block_size = int(mmap.PAGESIZE) / 1024
        # add mount point
        os.mkdir(self.mem_path)
        self.device = Partition(device="none", mountpoint=self.mem_path)
        self.device.mount(mountpoint=self.mem_path, fstype="tmpfs")
        free_space = (disk.freespace(self.mem_path)) / 1024
        # Leaving out some free space in tmpfs
        self.count = (free_space / self.block_size) - 3
    def setUp(self):
        '''
        Sets required params for dd workload and mounts the tmpfs
        '''

        # Get required mem info
        self.mem_path = os.path.join(data_dir.get_tmp_dir(), 'thp_space')
        self.block_size = int(mmap.PAGESIZE) / 1024
        # add mount point
        if os.path.exists(self.mem_path):
            os.makedirs(self.mem_path)
        self.device = Partition(device="none", mountpoint=self.mem_path)
        self.device.mount(mountpoint=self.mem_path, fstype="tmpfs")
        free_space = (disk.freespace(self.mem_path)) / 1024
        # Leaving out some free space in tmpfs
        self.count = (free_space / self.block_size) - 3
Example #12
0
    def _init_params(self):
        """
        Retrieves and checks the test params
        """
        self.disk = self.params.get('disk', default=None)
        self.dirs = self.params.get('dir', default=self.workdir)
        self.fstype = self.params.get('fs', default='ext4')

        memory_mb = memory.memtotal() / 1024
        self.chunk_mb = int(self.params.get('chunk_mb', default=None))
        if self.chunk_mb is None:   # By default total RAM
            self.chunk_mb = memory_mb
        if self.chunk_mb == 0:
            self.chunk_mb = 1
        if memory_mb > self.chunk_mb:
            self.cancel("Chunk size has to be greater or equal to RAM size. "
                        "(%s > %s)" % (self.chunk_mb, memory_mb))

        gigabytes = int(self.params.get('gigabytes', default=None))
        if gigabytes is None:
            free = 107374182400  # cap it at 100GB by default
            free = min(utils_disk.freespace(self.dirs) / 1073741824, free)
            gigabytes = free

        self.no_chunks = 1024 * gigabytes / self.chunk_mb
        if self.no_chunks == 0:
            self.cancel("Free disk space is lower than chunk size (%s, %s)"
                        % (1024 * gigabytes, self.chunk_mb))

        self.log.info("Test will use %s chunks %sMB each in %sMB RAM using %s "
                      "GB of disk space on %s dirs (%s).", self.no_chunks,
                      self.chunk_mb, memory_mb,
                      self.no_chunks * self.chunk_mb, len(self.dirs),
                      self.dirs)

        if self.disk is not None:
            self.part_obj = Partition(self.disk, mountpoint=self.dirs)
            self.log.info("Unmounting the disk/dir if it is already mounted")
            self.part_obj.unmount()
            self.log.info("creating %s fs on %s", self.fstype, self.disk)
            self.part_obj.mkfs(self.fstype)
            self.log.info("mounting %s on %s", self.disk, self.dirs)
            try:
                self.part_obj.mount()
            except PartitionError:
                self.fail("Mounting disk %s on directory %s failed"
                          % (self.disk, self.dirs))
Example #13
0
    def _init_params(self):
        """
        Retrieves and checks the test params
        """
        self.disk = self.params.get('disk', default=None)
        self.dirs = self.params.get('dir', default=self.workdir)
        self.fstype = self.params.get('fs', default='ext4')

        memory_mb = memory.memtotal() / 1024
        self.chunk_mb = int(self.params.get('chunk_mb', default=None))
        if self.chunk_mb is None:   # By default total RAM
            self.chunk_mb = memory_mb
        if self.chunk_mb == 0:
            self.chunk_mb = 1
        if memory_mb > self.chunk_mb:
            self.cancel("Chunk size has to be greater or equal to RAM size. "
                        "(%s > %s)" % (self.chunk_mb, memory_mb))

        gigabytes = int(self.params.get('gigabytes', default=None))
        if gigabytes is None:
            free = 107374182400  # cap it at 100GB by default
            free = min(utils_disk.freespace(self.dirs) / 1073741824, free)
            gigabytes = free

        self.no_chunks = 1024 * gigabytes / self.chunk_mb
        if self.no_chunks == 0:
            self.cancel("Free disk space is lower than chunk size (%s, %s)"
                        % (1024 * gigabytes, self.chunk_mb))

        self.log.info("Test will use %s chunks %sMB each in %sMB RAM using %s "
                      "GB of disk space on %s dirs (%s).", self.no_chunks,
                      self.chunk_mb, memory_mb,
                      self.no_chunks * self.chunk_mb, len(self.dirs),
                      self.dirs)

        if self.disk is not None:
            self.part_obj = Partition(self.disk, mountpoint=self.dirs)
            self.log.info("Unmounting the disk/dir if it is already mounted")
            self.part_obj.unmount()
            self.log.info("creating %s fs on %s", self.fstype, self.disk)
            self.part_obj.mkfs(self.fstype)
            self.log.info("mounting %s on %s", self.disk, self.dirs)
            try:
                self.part_obj.mount()
            except PartitionError:
                self.fail("Mounting disk %s on directory %s failed"
                          % (self.disk, self.dirs))
Example #14
0
    def setUp(self):
        """
        Downloads a copy of the linux kernel, calculate an estimated size of
        the uncompressed tarball, use this value to calculate the number of
        copies of the linux kernel that will be uncompressed.
        """
        self.nfail = 0
        self.tmpdir = data_dir.get_tmp_dir()
        self.tmpdir = self.params.get('dir_to_extract', default=self.tmpdir)
        self.base_dir = os.path.join(self.tmpdir, 'linux.orig')
        tarball_base = self.params.get('tarball_base',
                                       default='linux-2.6.18.tar.bz2')
        kernel_repo = self.params.get('kernel_repo',
                                      default='http://www.kernel.org/pub/'
                                      'linux/kernel/v2.6')
        tarball_url = os.path.join(kernel_repo, tarball_base)
        tarball_md5 = self.params.get('tarball_md5',
                                      default='296a6d150d260144639c3664d127d1'
                                      '74')
        parallel = self.params.get('parallel', default=True)
        self.parallel = parallel
        self.log.info('Downloading linux kernel tarball')
        self.tarball = self.fetch_asset(tarball_url,
                                        asset_hash=tarball_md5,
                                        algorithm='md5')
        size_tarball = os.path.getsize(self.tarball) // 1024 // 1024

        # Estimation of the tarball size after uncompression
        compress_ratio = 5
        est_size = size_tarball * compress_ratio
        self.sim_cps = self.get_sim_cps(est_size)
        self.log.info('Source file: %s', tarball_base)
        self.log.info('Megabytes per copy: %s', size_tarball)
        self.log.info('Compress ratio: %s', compress_ratio)
        self.log.info('Estimated size after uncompression: %s', est_size)
        self.log.info('Number of copies: %s', self.sim_cps)
        self.log.info('Parallel: %s', parallel)

        # Verify if space is available in disk
        disk_free_mb = (disk.freespace(self.tmpdir) // 1024) // 1024
        if disk_free_mb < (est_size * self.sim_cps):
            self.cancel("Space not available to extract the %s linux tars\n"
                        "Mount and Use other partitions in dir_to_extract arg "
                        "to run the test" % self.sim_cps)
    def setUp(self):
        """
        Downloads a copy of the linux kernel, calculate an estimated size of
        the uncompressed tarball, use this value to calculate the number of
        copies of the linux kernel that will be uncompressed.
        """
        self.nfail = 0
        self.tmpdir = data_dir.get_tmp_dir()
        self.tmpdir = self.params.get('dir_to_extract', default=self.tmpdir)
        tarball_base = self.params.get('tarball_base',
                                       default='linux-2.6.18.tar.bz2')
        kernel_repo = self.params.get('kernel_repo',
                                      default='http://www.kernel.org/pub/'
                                      'linux/kernel/v2.6')
        tarball_url = os.path.join(kernel_repo, tarball_base)
        tarball_md5 = self.params.get('tarball_md5',
                                      default='296a6d150d260144639c3664d127d1'
                                      '74')
        parallel = self.params.get('parallel', default=True)
        self.parallel = parallel
        self.log.info('Downloading linux kernel tarball')
        self.tarball = self.fetch_asset(tarball_url, asset_hash=tarball_md5,
                                        algorithm='md5')
        size_tarball = os.path.getsize(self.tarball) / 1024 / 1024

        # Estimation of the tarball size after uncompression
        compress_ratio = 5
        est_size = size_tarball * compress_ratio
        self.sim_cps = self.get_sim_cps(est_size)
        self.log.info('Source file: %s', tarball_base)
        self.log.info('Megabytes per copy: %s', size_tarball)
        self.log.info('Compress ratio: %s', compress_ratio)
        self.log.info('Estimated size after uncompression: %s', est_size)
        self.log.info('Number of copies: %s', self.sim_cps)
        self.log.info('Parallel: %s', parallel)

        # Verify if space is available in disk
        disk_free_mb = (disk.freespace(self.tmpdir) / 1024) / 1024
        if (disk_free_mb < est_size * self.sim_cps):
            self.cancel("Space not available to extract the %s linux tars\n"
                        "Mount and Use other partitions in dir_to_extract arg "
                        "to run the test" % self.sim_cps)
    def _init_params(self):
        """
        Retrieves and checks the test params
        """
        disks = self.params.get('disks', default=None)
        if disks is None:   # Avocado does not accept lists in params.get()
            disks = [self.workdir]
        elif isinstance(disks, basestring):  # Allow specifying disks as str
            disks = disks.split(',')    # it's string pylint: disable=E1101
        for disk in disks:  # Disks have to be mounted dirs
            if not os.path.isdir(disk):
                os.makedirs(disk)
        self.disks = disks

        memory_mb = memory.memtotal() / 1024
        self.chunk_mb = self.params.get('chunk_mb', default=None)
        if self.chunk_mb is None:   # By default total RAM
            self.chunk_mb = memory_mb
        if self.chunk_mb == 0:
            self.chunk_mb = 1
        if memory_mb > self.chunk_mb:
            self.skip("Chunk size has to be greater or equal to RAM size. "
                      "(%s > %s)" % (self.chunk_mb, memory_mb))

        gigabytes = self.params.get('gigabytes', default=None)
        if gigabytes is None:
            free = 100  # cap it at 100GB by default
            for disk in self.disks:
                free = min(utils_disk.freespace(disk) / 1073741824, free)
            gigabytes = free

        self.no_chunks = 1024 * gigabytes / self.chunk_mb
        if self.no_chunks == 0:
            self.skip("Free disk space is lower than chunk size (%s, %s)"
                      % (1024 * gigabytes, self.chunk_mb))

        self.log.info("Test will use %s chunks %sMB each in %sMB RAM using %s "
                      "GB of disk space on %s disks (%s).", self.no_chunks,
                      self.chunk_mb, memory_mb,
                      self.no_chunks * self.chunk_mb, len(self.disks),
                      self.disks)
Example #17
0
    def setUp(self):
        '''
        Sets required params for dd workload and mounts the tmpfs
        '''

        # Get required mem info
        self.mem_path = os.path.join(data_dir.get_tmp_dir(), 'thp_space')
        self.block_size = int(mmap.PAGESIZE) // 1024
        # add mount point
        if os.path.exists(self.mem_path):
            os.makedirs(self.mem_path)
        self.device = Partition(device="none", mountpoint=self.mem_path)
        self.device.mount(mountpoint=self.mem_path,
                          fstype="tmpfs",
                          mnt_check=False)
        free_space = (disk.freespace(self.mem_path)) // 1024
        # Reserving some memory (out of 100% memory reserving 25% memory)
        res_free_space = int(free_space / 4)
        free_space = free_space - res_free_space
        # Leaving out some free space in tmpfs
        self.count = (free_space // self.block_size) - 3
Example #18
0
    def setUp(self):
        """
        Build xfstest
        Source: git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git
        """
        self.use_dd = False
        root_fs = process.system_output(
            "df -T / | awk 'END {print $2}'", shell=True)
        if root_fs in ['ext3', 'ext4']:
            self.use_dd = True
        sm = SoftwareManager()

        self.detected_distro = distro.detect()

        packages = ['e2fsprogs', 'automake', 'gcc', 'quota', 'attr',
                    'make', 'xfsprogs', 'gawk']
        if 'Ubuntu' in self.detected_distro.name:
            packages.extend(
                ['xfslibs-dev', 'uuid-dev', 'libuuid1',
                 'libattr1-dev', 'libacl1-dev', 'libgdbm-dev',
                 'uuid-runtime', 'libaio-dev', 'fio', 'dbench', 'btrfs-tools'])
            if '14' in self.detected_distro.version:
                packages.extend(['libtool'])
            elif '18' in self.detected_distro.version:
                packages.extend(['libtool-bin', 'libgdbm-compat-dev'])
            else:
                packages.extend(['libtool-bin'])

        # FIXME: "redhat" as the distro name for RHEL is deprecated
        # on Avocado versions >= 50.0.  This is a temporary compatibility
        # enabler for older runners, but should be removed soon
        elif self.detected_distro.name in ['centos', 'fedora', 'rhel',
                                           'redhat', 'SuSE']:
            packages.extend(['acl', 'bc', 'dump', 'indent', 'libtool', 'lvm2',
                             'xfsdump', 'psmisc', 'sed', 'libacl-devel',
                             'libattr-devel', 'libaio-devel', 'libuuid-devel',
                             'openssl-devel', 'xfsprogs-devel'])
            if self.detected_distro.name == 'SuSE':
                packages.extend(['libbtrfs-devel', 'libcap-progs'])
            else:
                packages.extend(['btrfs-progs-devel'])

            if self.detected_distro.name in ['centos', 'fedora']:
                packages.extend(['fio', 'dbench'])
        else:
            self.cancel("test not supported in %s" % self.detected_distro.name)

        for package in packages:
            if not sm.check_installed(package) and not sm.install(package):
                self.cancel("Fail to install %s required for this test." %
                            package)

        self.skip_dangerous = self.params.get('skip_dangerous', default=True)
        self.test_range = self.params.get('test_range', default=None)
        self.scratch_mnt = self.params.get(
            'scratch_mnt', default='/mnt/scratch')
        self.test_mnt = self.params.get('test_mnt', default='/mnt/test')
        self.disk_mnt = self.params.get('disk_mnt', default='/mnt/loop_device')
        self.dev_type = self.params.get('type', default='loop')
        self.fs_to_test = self.params.get('fs', default='ext4')
        if process.system('which mkfs.%s' % self.fs_to_test,
                          ignore_status=True):
            self.cancel('Unknown filesystem %s' % self.fs_to_test)
        mount = True
        self.devices = []
        shutil.copyfile(self.get_data('local.config'),
                        os.path.join(self.teststmpdir, 'local.config'))
        shutil.copyfile(self.get_data('group'),
                        os.path.join(self.teststmpdir, 'group'))

        if self.dev_type == 'loop':
            base_disk = self.params.get('disk', default=None)
            loop_size = self.params.get('loop_size', default='7GiB')
            if not base_disk:
                # Using root for file creation by default
                check = (int(loop_size.split('GiB')[0]) * 2) + 1
                if disk.freespace('/') / 1073741824 > check:
                    self.disk_mnt = ''
                    mount = False
                else:
                    self.cancel('Need %s GB to create loop devices' % check)
            self._create_loop_device(base_disk, loop_size, mount)
        else:
            self.test_dev = self.params.get('disk_test', default=None)
            self.scratch_dev = self.params.get('disk_scratch', default=None)
            self.devices.extend([self.test_dev, self.scratch_dev])
        # mkfs for devices
        if self.devices:
            cfg_file = os.path.join(self.teststmpdir, 'local.config')
            with open(cfg_file, "r") as sources:
                lines = sources.readlines()
            with open(cfg_file, "w") as sources:
                for line in lines:
                    if line.startswith('export TEST_DEV'):
                        sources.write(
                            re.sub(r'export TEST_DEV=.*', 'export TEST_DEV=%s'
                                   % self.devices[0], line))
                    elif line.startswith('export TEST_DIR'):
                        sources.write(
                            re.sub(r'export TEST_DIR=.*', 'export TEST_DIR=%s'
                                   % self.test_mnt, line))
                    elif line.startswith('export SCRATCH_DEV'):
                        sources.write(re.sub(
                            r'export SCRATCH_DEV=.*', 'export SCRATCH_DEV=%s'
                                                      % self.devices[1], line))
                    elif line.startswith('export SCRATCH_MNT'):
                        sources.write(
                            re.sub(
                                r'export SCRATCH_MNT=.*',
                                'export SCRATCH_MNT=%s' %
                                self.scratch_mnt,
                                line))
                        break

            for dev in self.devices:
                dev_obj = partition.Partition(dev)
                dev_obj.mkfs(fstype=self.fs_to_test)

        git.get_repo('git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git',
                     destination_dir=self.teststmpdir)

        build.make(self.teststmpdir)
        self.available_tests = self._get_available_tests()

        self.test_list = self._create_test_list(self.test_range)
        self.log.info("Tests available in srcdir: %s",
                      ", ".join(self.available_tests))
        if not self.test_range:
            self.exclude = self.params.get('exclude', default=None)
            self.gen_exclude = self.params.get('gen_exclude', default=None)
            self.share_exclude = self.params.get('share_exclude', default=None)
            if self.exclude or self.gen_exclude or self.share_exclude:
                self.exclude_file = os.path.join(self.teststmpdir, 'exclude')
                if self.exclude:
                    self._create_test_list(self.exclude, self.fs_to_test,
                                           dangerous=False)
                if self.gen_exclude:
                    self._create_test_list(self.gen_exclude, "generic",
                                           dangerous=False)
                if self.share_exclude:
                    self._create_test_list(self.share_exclude, "shared",
                                           dangerous=False)
        if self.detected_distro.name is not 'SuSE':
            process.run('useradd 123456-fsgqa', sudo=True)
            process.run('useradd fsgqa', sudo=True)
        else:
            process.run('useradd -m -U fsgqa', sudo=True)
            process.run('groupadd sys', sudo=True)
        if not os.path.exists(self.scratch_mnt):
            os.makedirs(self.scratch_mnt)
        if not os.path.exists(self.test_mnt):
            os.makedirs(self.test_mnt)
    def test(self):
        self.plib.enable_region()
        regions = self.plib.run_ndctl_list('-R')
        self.plib.destroy_namespace(force=True)

        region = self.plib.run_ndctl_list_val(regions[0], 'dev')
        split = self.params.get('split_ns', default=False)
        if len(regions) == 1:
            if self.plib.is_region_legacy(region):
                self.cancel("Cannot create DM with single pmem device")
            if not split:
                self.cancel("Cannot run test without split option enabled")

        if split:
            if self.plib.is_region_legacy(region):
                self.cancel("Cannot split pmem device on legacy hardware")

            size_align = self.get_size_alignval()

            self.log.info("Creating namespace with existing regions")
            for reg_json in regions:
                region = self.plib.run_ndctl_list_val(reg_json, 'dev')
                slot_count = self.plib.get_slot_count(region)
                reg_size = self.plib.run_ndctl_list_val(
                    self.plib.run_ndctl_list('-r %s' % region)[0], 'size')
                namespace_size = reg_size // slot_count
                # Now align the namespace size
                namespace_size = (namespace_size //
                                  size_align) * size_align
                if namespace_size <= size_align:
                    self.log.warn("Skipping namespace size less than pagesize")
                    continue
                for _ in range(0, slot_count):
                    self.plib.create_namespace(
                        region=region, size=namespace_size)
        else:
            self.log.info("Creating namespace with full size")
            for reg_json in regions:
                region = self.plib.run_ndctl_list_val(reg_json, 'dev')
                self.plib.create_namespace(region=region)
        devices = self.plib.run_ndctl_list('-N')
        blk_cmd = ""
        bdev = None
        blk_size1 = 0
        for cnt, dev in enumerate(devices):
            bdev = self.plib.run_ndctl_list_val(dev, 'blockdev')
            bdev = "/dev/%s" % bdev
            blk_size2 = process.system_output(
                "blockdev --getsz %s" % bdev).decode()
            blk_cmd += ' %s %s linear %s 0 "\\\\n"' % (
                blk_size1, blk_size2, bdev)
            blk_size1 += int(blk_size2)
            if cnt == len(devices) - 1:
                break
        dm_cmd = 'echo -e "%s" | dmsetup create linear-pmem' % blk_cmd
        if process.system(dm_cmd, shell=True, sudo=True, ignore_status=True):
            self.fail("Creating DM failed")
        self.log.info("Running FIO on device-mapper")
        dm_disk = "/dev/mapper/linear-pmem"
        self.part = partition.Partition(dm_disk)
        self.part.mkfs(fstype='xfs', args='-b size=%s -s size=512 -m reflink=0' %
                       memory.get_page_size())
        mnt_path = self.params.get('mnt_point', default='/pmem')
        if not os.path.exists(mnt_path):
            os.makedirs(mnt_path)
        self.part.mount(mountpoint=mnt_path, args='-o dax')
        self.log.info("Test will run on %s", mnt_path)
        fio_job = self.params.get('fio_job', default='ndctl-fio.job')
        size = disk.freespace(mnt_path) * 0.9
        cmd = '%s --directory %s --filename mmap-pmem --size %s %s' % (
            self.build_fio(), mnt_path, size, self.get_data(fio_job))
        if process.system(cmd, ignore_status=True):
            self.fail("FIO mmap workload on fsdax failed")
Example #20
0
    def setUp(self):
        """
        Build xfstest
        Source: git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git
        """
        self.use_dd = False
        root_fs = process.system_output(
            "df -T / | awk 'END {print $2}'", shell=True).decode("utf-8")
        if root_fs in ['ext3', 'ext4']:
            self.use_dd = True
        self.dev_type = self.params.get('type', default='loop')

        sm = SoftwareManager()

        self.detected_distro = distro.detect()

        packages = ['e2fsprogs', 'automake', 'gcc', 'quota', 'attr',
                    'make', 'xfsprogs', 'gawk']
        if self.detected_distro.name in ['Ubuntu', 'debian']:
            packages.extend(
                ['xfslibs-dev', 'uuid-dev', 'libuuid1',
                 'libattr1-dev', 'libacl1-dev', 'libgdbm-dev',
                 'uuid-runtime', 'libaio-dev', 'fio', 'dbench',
                 'gettext', 'libinih-dev', 'liburcu-dev', 'libblkid-dev',
                 'liblzo2-dev', 'zlib1g-dev', 'e2fslibs-dev', 'asciidoc',
                 'xmlto', 'libzstd-dev', 'libudev-dev'])
            if self.detected_distro.version in ['14']:
                packages.extend(['libtool'])
            elif self.detected_distro.version in ['18', '20']:
                packages.extend(['libtool-bin', 'libgdbm-compat-dev'])
            else:
                packages.extend(['libtool-bin'])

        elif self.detected_distro.name in ['centos', 'fedora', 'rhel', 'SuSE']:
            if self.dev_type == 'nvdimm':
                packages.extend(['ndctl', 'parted'])
                if self.detected_distro.name == 'rhel':
                    packages.extend(['daxctl'])
            packages.extend(['acl', 'bc', 'dump', 'indent', 'libtool', 'lvm2',
                             'xfsdump', 'psmisc', 'sed', 'libacl-devel',
                             'libattr-devel', 'libaio-devel', 'libuuid-devel',
                             'openssl-devel', 'xfsprogs-devel', 'gettext',
                             'libblkid-devel', 'lzo-devel', 'zlib-devel',
                             'e2fsprogs-devel', 'asciidoc', 'xmlto',
                             'libzstd-devel', 'systemd-devel', 'meson',
                             'gcc-c++'])

            if self.detected_distro.name == 'SuSE':
                packages.extend(['libbtrfs-devel', 'libcap-progs',
                                'liburcu-devel', 'libinih-devel'])
            else:
                packages.extend(['btrfs-progs-devel', 'userspace-rcu-devel'])

            packages_remove = ['indent', 'btrfs-progs-devel']
            if self.detected_distro.name == 'rhel' and\
                    self.detected_distro.version.startswith('8'):
                packages = list(set(packages)-set(packages_remove))
            elif self.detected_distro.name == 'rhel' and\
                    self.detected_distro.version.startswith('9'):
                packages_remove.extend(['dump'])
                packages = list(set(packages)-set(packages_remove))

            if self.detected_distro.name in ['centos', 'fedora']:
                packages.extend(['fio', 'dbench'])
        else:
            self.cancel("test not supported in %s" % self.detected_distro.name)

        for package in packages:
            if not sm.check_installed(package) and not sm.install(package):
                self.cancel("Fail to install %s required for this test." %
                            package)
        self.skip_dangerous = self.params.get('skip_dangerous', default=True)
        self.test_range = self.params.get('test_range', default=None)
        self.scratch_mnt = self.params.get(
            'scratch_mnt', default='/mnt/scratch')
        self.test_mnt = self.params.get('test_mnt', default='/mnt/test')
        self.disk_mnt = self.params.get('disk_mnt', default='/mnt/loop_device')
        self.fs_to_test = self.params.get('fs', default='ext4')
        self.run_type = self.params.get('run_type', default='distro')

        if self.run_type == 'upstream':
            prefix = "/usr/local"
            bin_prefix = "/usr/local/bin"

            if self.detected_distro.name == 'SuSE':
                # SuSE has /sbin at a higher priority than /usr/local/bin
                # in $PATH, so install all the binaries in /sbin to make
                # sure they are picked up correctly by xfstests.
                #
                # We still install in /usr/local but binaries are kept in
                # /sbin
                bin_prefix = "/sbin"

            if self.fs_to_test == "ext4":
                # Build e2fs progs
                e2fsprogs_dir = os.path.join(self.teststmpdir, 'e2fsprogs')
                if not os.path.exists(e2fsprogs_dir):
                    os.makedirs(e2fsprogs_dir)
                e2fsprogs_url = self.params.get('e2fsprogs_url')
                git.get_repo(e2fsprogs_url, destination_dir=e2fsprogs_dir)
                e2fsprogs_build_dir = os.path.join(e2fsprogs_dir, 'build')
                if not os.path.exists(e2fsprogs_build_dir):
                    os.makedirs(e2fsprogs_build_dir)
                os.chdir(e2fsprogs_build_dir)
                process.run("../configure --prefix=%s --bindir=%s --sbindir=%s"
                            % (prefix, bin_prefix, bin_prefix), verbose=True)
                build.make(e2fsprogs_build_dir)
                build.make(e2fsprogs_build_dir, extra_args='install')

            if self.fs_to_test == "xfs":
                if self.detected_distro.name in ['centos', 'fedora', 'rhel']:
                    libini_path = process.run("ldconfig -p | grep libini",
                                              verbose=True, ignore_status=True)
                    if not libini_path:
                        # Build libini.h as it is needed for xfsprogs
                        libini_dir = os.path.join(self.teststmpdir, 'libini')
                        if not os.path.exists(libini_dir):
                            os.makedirs(libini_dir)
                        git.get_repo('https://github.com/benhoyt/inih',
                                     destination_dir=libini_dir)
                        os.chdir(libini_dir)
                        process.run("meson build", verbose=True)
                        libini_build_dir = os.path.join(libini_dir, 'build')
                        if os.path.exists(libini_build_dir):
                            os.chdir(libini_build_dir)
                            process.run("meson install", verbose=True)
                        else:
                            self.fail('Something went wrong while building \
                                      libini. Please check the logs.')
                # Build xfs progs
                xfsprogs_dir = os.path.join(self.teststmpdir, 'xfsprogs')
                if not os.path.exists(xfsprogs_dir):
                    os.makedirs(xfsprogs_dir)
                xfsprogs_url = self.params.get('xfsprogs_url')
                git.get_repo(xfsprogs_url, destination_dir=xfsprogs_dir)
                os.chdir(xfsprogs_dir)
                build.make(xfsprogs_dir)
                process.run("./configure --prefix=%s --bindir=%s --sbindir=%s"
                            % (prefix, bin_prefix, bin_prefix), verbose=True)
                build.make(xfsprogs_dir, extra_args='install')

            if self.fs_to_test == "btrfs":
                # Build btrfs progs
                btrfsprogs_dir = os.path.join(self.teststmpdir, 'btrfsprogs')
                if not os.path.exists(btrfsprogs_dir):
                    os.makedirs(btrfsprogs_dir)
                btrfsprogs_url = self.params.get('btrfsprogs_url')
                git.get_repo(btrfsprogs_url, destination_dir=btrfsprogs_dir)
                os.chdir(btrfsprogs_dir)
                process.run("./autogen.sh", verbose=True)
                process.run("./configure --prefix=%s --bindir=%s --sbindir=%s --disable-documentation"
                            % (prefix, bin_prefix, bin_prefix), verbose=True)
                build.make(btrfsprogs_dir)
                build.make(btrfsprogs_dir, extra_args='install')

        # Check versions of fsprogs
        fsprogs_ver = process.system_output("mkfs.%s -V" % self.fs_to_test,
                                            ignore_status=True,
                                            shell=True).decode("utf-8")
        self.log.info(fsprogs_ver)

        if process.system('which mkfs.%s' % self.fs_to_test,
                          ignore_status=True):
            self.cancel('Unknown filesystem %s' % self.fs_to_test)
        mount = True
        self.devices = []
        self.log_devices = []
        shutil.copyfile(self.get_data('local.config'),
                        os.path.join(self.teststmpdir, 'local.config'))
        shutil.copyfile(self.get_data('group'),
                        os.path.join(self.teststmpdir, 'group'))

        self.log_test = self.params.get('log_test', default='')
        self.log_scratch = self.params.get('log_scratch', default='')

        if self.dev_type == 'loop':
            base_disk = self.params.get('disk', default=None)
            loop_size = self.params.get('loop_size', default='7GiB')
            if not base_disk:
                # Using root for file creation by default
                check = (int(loop_size.split('GiB')[0]) * 2) + 1
                if disk.freespace('/') / 1073741824 > check:
                    self.disk_mnt = ''
                    mount = False
                else:
                    self.cancel('Need %s GB to create loop devices' % check)
            self._create_loop_device(base_disk, loop_size, mount)
        elif self.dev_type == 'nvdimm':
            self.setup_nvdimm()
        else:
            self.test_dev = self.params.get('disk_test', default=None)
            self.scratch_dev = self.params.get('disk_scratch', default=None)
            self.devices.extend([self.test_dev, self.scratch_dev])
        # mkfs for devices
        if self.devices:
            cfg_file = os.path.join(self.teststmpdir, 'local.config')
            self.mkfs_opt = self.params.get('mkfs_opt', default='')
            self.mount_opt = self.params.get('mount_opt', default='')
            with open(cfg_file, "r") as sources:
                lines = sources.readlines()
            with open(cfg_file, "w") as sources:
                for line in lines:
                    if line.startswith('export TEST_DEV'):
                        sources.write(
                            re.sub(r'export TEST_DEV=.*', 'export TEST_DEV=%s'
                                   % self.devices[0], line))
                    elif line.startswith('export TEST_DIR'):
                        sources.write(
                            re.sub(r'export TEST_DIR=.*', 'export TEST_DIR=%s'
                                   % self.test_mnt, line))
                    elif line.startswith('export SCRATCH_DEV'):
                        sources.write(re.sub(
                            r'export SCRATCH_DEV=.*', 'export SCRATCH_DEV=%s'
                                                      % self.devices[1], line))
                    elif line.startswith('export SCRATCH_MNT'):
                        sources.write(
                            re.sub(
                                r'export SCRATCH_MNT=.*',
                                'export SCRATCH_MNT=%s' %
                                self.scratch_mnt,
                                line))
                        break
            with open(cfg_file, "a") as sources:
                if self.log_test:
                    sources.write('export USE_EXTERNAL=yes\n')
                    sources.write('export TEST_LOGDEV="%s"\n' % self.log_test)
                    self.log_devices.append(self.log_test)
                if self.log_scratch:
                    sources.write('export SCRATCH_LOGDEV="%s"\n' %
                                  self.log_scratch)
                    self.log_devices.append(self.log_scratch)
                if self.mkfs_opt:
                    sources.write('MKFS_OPTIONS="%s"\n' % self.mkfs_opt)
                if self.mount_opt:
                    sources.write('MOUNT_OPTIONS="%s"\n' % self.mount_opt)
            self.logdev_opt = self.params.get('logdev_opt', default='')
            for dev in self.log_devices:
                dev_obj = partition.Partition(dev)
                dev_obj.mkfs(fstype=self.fs_to_test, args=self.mkfs_opt)
            for ite, dev in enumerate(self.devices):
                dev_obj = partition.Partition(dev)
                if self.logdev_opt:
                    dev_obj.mkfs(fstype=self.fs_to_test, args='%s %s=%s' % (
                        self.mkfs_opt, self.logdev_opt, self.log_devices[ite]))
                else:
                    dev_obj.mkfs(fstype=self.fs_to_test, args=self.mkfs_opt)

        git.get_repo('git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git',
                     destination_dir=self.teststmpdir)

        build.make(self.teststmpdir)
        self.available_tests = self._get_available_tests()

        self.test_list = self._create_test_list(self.test_range)
        self.log.info("Tests available in srcdir: %s",
                      ", ".join(self.available_tests))
        if not self.test_range:
            self.exclude = self.params.get('exclude', default=None)
            self.gen_exclude = self.params.get('gen_exclude', default=None)
            self.share_exclude = self.params.get('share_exclude', default=None)
            if self.exclude or self.gen_exclude or self.share_exclude:
                self.exclude_file = os.path.join(self.teststmpdir, 'exclude')
                if self.exclude:
                    self._create_test_list(self.exclude, self.fs_to_test,
                                           dangerous=False)
                if self.gen_exclude:
                    self._create_test_list(self.gen_exclude, "generic",
                                           dangerous=False)
                if self.share_exclude:
                    self._create_test_list(self.share_exclude, "shared",
                                           dangerous=False)
        if self.detected_distro.name is not 'SuSE':
            if process.system('useradd 123456-fsgqa', sudo=True, ignore_status=True):
                self.log.warn('useradd 123456-fsgqa failed')
            if process.system('useradd fsgqa', sudo=True, ignore_status=True):
                self.log.warn('useradd fsgqa failed')
        else:
            if process.system('useradd -m -U fsgqa', sudo=True, ignore_status=True):
                self.log.warn('useradd fsgqa failed')
            if process.system('groupadd sys', sudo=True, ignore_status=True):
                self.log.warn('groupadd sys failed')
        if not os.path.exists(self.scratch_mnt):
            os.makedirs(self.scratch_mnt)
        if not os.path.exists(self.test_mnt):
            os.makedirs(self.test_mnt)
    def setUp(self):
        """
        Build xfstest
        Source: git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git
        """
        self.use_dd = False
        root_fs = process.system_output("df -T / | awk 'END {print $2}'",
                                        shell=True).decode("utf-8")
        if root_fs in ['ext3', 'ext4']:
            self.use_dd = True
        self.dev_type = self.params.get('type', default='loop')

        sm = SoftwareManager()

        self.detected_distro = distro.detect()

        packages = [
            'e2fsprogs', 'automake', 'gcc', 'quota', 'attr', 'make',
            'xfsprogs', 'gawk'
        ]
        if self.detected_distro.name in ['Ubuntu', 'debian']:
            packages.extend([
                'xfslibs-dev', 'uuid-dev', 'libuuid1', 'libattr1-dev',
                'libacl1-dev', 'libgdbm-dev', 'uuid-runtime', 'libaio-dev',
                'fio', 'dbench', 'libbtrfs-dev'
            ])
            if self.detected_distro.version in ['14']:
                packages.extend(['libtool'])
            elif self.detected_distro.version in ['18', '20']:
                packages.extend(['libtool-bin', 'libgdbm-compat-dev'])
            else:
                packages.extend(['libtool-bin'])

        # FIXME: "redhat" as the distro name for RHEL is deprecated
        # on Avocado versions >= 50.0.  This is a temporary compatibility
        # enabler for older runners, but should be removed soon
        elif self.detected_distro.name in ['centos', 'fedora', 'rhel', 'SuSE']:
            if self.dev_type == 'nvdimm':
                packages.extend(['ndctl', 'parted'])
                if self.detected_distro.name == 'rhel':
                    packages.extend(['daxctl'])
            packages.extend([
                'acl', 'bc', 'dump', 'indent', 'libtool', 'lvm2', 'xfsdump',
                'psmisc', 'sed', 'libacl-devel', 'libattr-devel',
                'libaio-devel', 'libuuid-devel', 'openssl-devel',
                'xfsprogs-devel'
            ])

            if self.detected_distro.name == 'SuSE':
                packages.extend(['libbtrfs-devel', 'libcap-progs'])
            else:
                packages.extend(['btrfs-progs-devel'])

            if self.detected_distro.name == 'rhel' and\
                    self.detected_distro.version.startswith('8'):
                packages.remove('indent')
                packages.remove('btrfs-progs-devel')

            if self.detected_distro.name in ['centos', 'fedora']:
                packages.extend(['fio', 'dbench'])
        else:
            self.cancel("test not supported in %s" % self.detected_distro.name)

        for package in packages:
            if not sm.check_installed(package) and not sm.install(package):
                self.cancel("Fail to install %s required for this test." %
                            package)
        self.skip_dangerous = self.params.get('skip_dangerous', default=True)
        self.test_range = self.params.get('test_range', default=None)
        self.scratch_mnt = self.params.get('scratch_mnt',
                                           default='/mnt/scratch')
        self.test_mnt = self.params.get('test_mnt', default='/mnt/test')
        self.disk_mnt = self.params.get('disk_mnt', default='/mnt/loop_device')
        self.fs_to_test = self.params.get('fs', default='ext4')

        if process.system('which mkfs.%s' % self.fs_to_test,
                          ignore_status=True):
            self.cancel('Unknown filesystem %s' % self.fs_to_test)
        mount = True
        self.devices = []
        self.log_devices = []
        shutil.copyfile(self.get_data('local.config'),
                        os.path.join(self.teststmpdir, 'local.config'))
        shutil.copyfile(self.get_data('group'),
                        os.path.join(self.teststmpdir, 'group'))

        if self.dev_type == 'loop':
            base_disk = self.params.get('disk', default=None)
            loop_size = self.params.get('loop_size', default='7GiB')
            if not base_disk:
                # Using root for file creation by default
                check = (int(loop_size.split('GiB')[0]) * 2) + 1
                if disk.freespace('/') / 1073741824 > check:
                    self.disk_mnt = ''
                    mount = False
                else:
                    self.cancel('Need %s GB to create loop devices' % check)
            self._create_loop_device(base_disk, loop_size, mount)
        elif self.dev_type == 'nvdimm':
            self.test_dev = self.params.get('disk_test', default=None)
            self.scratch_dev = self.params.get('disk_scratch', default=None)
            self.log_test = self.params.get('log_test', default='')
            self.log_scratch = self.params.get('log_scratch', default='')
            logflag = self.params.get('logdev', default=False)
            self.plib = pmem.PMem()
            regions = sorted(self.plib.run_ndctl_list('-R'),
                             key=lambda i: i['size'],
                             reverse=True)
            if not (self.test_dev and self.scratch_dev):
                if not regions:
                    self.plib.enable_region()
                    regions = sorted(self.plib.run_ndctl_list('-R'),
                                     key=lambda i: i['size'],
                                     reverse=True)
                region = self.plib.run_ndctl_list_val(regions[0], 'dev')
                if not self.plib.is_region_legacy(region):
                    self.plib.destroy_namespace(region=region, force=True)
                    self.plib.create_namespace(region=region,
                                               size='21G',
                                               sector_size='512')
                    pmem_dev = self.plib.run_ndctl_list_val(
                        self.plib.run_ndctl_list('-N -r %s' % region)[0],
                        'blockdev')
                    if logflag:
                        if not (self.log_test and self.log_scratch):
                            self.plib.create_namespace(region=region,
                                                       size='3G',
                                                       mode='sector',
                                                       sector_size='512')
                            log_dev = self.plib.run_ndctl_list_val(
                                self.plib.run_ndctl_list('-N -r %s -m sector' %
                                                         region)[0],
                                'blockdev')
                else:
                    self.plib.enable_region()
                    if not len(regions) > 1 and logflag:
                        self.cancel("Cannot use logdev with one region")
                    else:
                        reg_2 = self.plib.run_ndctl_list_val(regions[1], 'dev')
                        self.plib.create_namespace(region=reg_2,
                                                   size='3G',
                                                   mode='sector',
                                                   sector_size='512')
                        log_dev = self.plib.run_ndctl_list_val(
                            self.plib.run_ndctl_list(' -N -r %s -m sector' %
                                                     reg_2)[0], 'blockdev')
                    pmem_dev = self.plib.run_ndctl_list_val(
                        self.plib.run_ndctl_list('-N -r %s' % region)[0],
                        'blockdev')
                if process.system('parted -s -a optimal /dev/%s mklabel gpt --'
                                  ' mkpart primary xfs 1MiB 10GiB mkpart '
                                  'primary xfs 10GiB 20GiB' % pmem_dev,
                                  shell=True):
                    self.cancel("Failed to setup PMEM partitions")
                self.test_dev = "/dev/%sp1" % pmem_dev
                self.scratch_dev = "/dev/%sp2" % pmem_dev
            self.devices.extend([self.test_dev, self.scratch_dev])
            if logflag and not (self.log_test and self.log_scratch):
                if process.system('parted -s -a optimal /dev/%s mklabel gpt --'
                                  ' mkpart primary xfs 1MiB 1GiB mkpart '
                                  'primary xfs 1GiB 2GiB' % log_dev,
                                  shell=True):
                    self.cancel("Failed to setup logdev partitions")
                self.log_test = "/dev/%s1" % log_dev
                self.log_scratch = "/dev/%s2" % log_dev
        else:
            self.test_dev = self.params.get('disk_test', default=None)
            self.scratch_dev = self.params.get('disk_scratch', default=None)
            self.devices.extend([self.test_dev, self.scratch_dev])
        # mkfs for devices
        if self.devices:
            cfg_file = os.path.join(self.teststmpdir, 'local.config')
            self.mkfs_opt = self.params.get('mkfs_opt', default='')
            self.mount_opt = self.params.get('mount_opt', default='')
            with open(cfg_file, "r") as sources:
                lines = sources.readlines()
            with open(cfg_file, "w") as sources:
                for line in lines:
                    if line.startswith('export TEST_DEV'):
                        sources.write(
                            re.sub(r'export TEST_DEV=.*',
                                   'export TEST_DEV=%s' % self.devices[0],
                                   line))
                    elif line.startswith('export TEST_DIR'):
                        sources.write(
                            re.sub(r'export TEST_DIR=.*',
                                   'export TEST_DIR=%s' % self.test_mnt, line))
                    elif line.startswith('export SCRATCH_DEV'):
                        sources.write(
                            re.sub(r'export SCRATCH_DEV=.*',
                                   'export SCRATCH_DEV=%s' % self.devices[1],
                                   line))
                    elif line.startswith('export SCRATCH_MNT'):
                        sources.write(
                            re.sub(r'export SCRATCH_MNT=.*',
                                   'export SCRATCH_MNT=%s' % self.scratch_mnt,
                                   line))
                        break
            with open(cfg_file, "a") as sources:
                if self.log_test:
                    sources.write('export USE_EXTERNAL=yes\n')
                    sources.write('export TEST_LOGDEV="%s"\n' % self.log_test)
                    self.log_devices.append(self.log_test)
                if self.log_scratch:
                    sources.write('export SCRATCH_LOGDEV="%s"\n' %
                                  self.log_scratch)
                    self.log_devices.append(self.log_scratch)
                if self.mkfs_opt:
                    sources.write('MKFS_OPTIONS="%s"\n' % self.mkfs_opt)
                if self.mount_opt:
                    sources.write('MOUNT_OPTIONS="%s"\n' % self.mount_opt)
            self.logdev_opt = self.params.get('logdev_opt', default='')
            for dev in self.log_devices:
                dev_obj = partition.Partition(dev)
                dev_obj.mkfs(fstype=self.fs_to_test, args=self.mkfs_opt)
            for ite, dev in enumerate(self.devices):
                dev_obj = partition.Partition(dev)
                if self.logdev_opt:
                    dev_obj.mkfs(fstype=self.fs_to_test,
                                 args='%s %s=%s' %
                                 (self.mkfs_opt, self.logdev_opt,
                                  self.log_devices[ite]))
                else:
                    dev_obj.mkfs(fstype=self.fs_to_test, args=self.mkfs_opt)

        git.get_repo('git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git',
                     destination_dir=self.teststmpdir)

        build.make(self.teststmpdir)
        self.available_tests = self._get_available_tests()

        self.test_list = self._create_test_list(self.test_range)
        self.log.info("Tests available in srcdir: %s",
                      ", ".join(self.available_tests))
        if not self.test_range:
            self.exclude = self.params.get('exclude', default=None)
            self.gen_exclude = self.params.get('gen_exclude', default=None)
            self.share_exclude = self.params.get('share_exclude', default=None)
            if self.exclude or self.gen_exclude or self.share_exclude:
                self.exclude_file = os.path.join(self.teststmpdir, 'exclude')
                if self.exclude:
                    self._create_test_list(self.exclude,
                                           self.fs_to_test,
                                           dangerous=False)
                if self.gen_exclude:
                    self._create_test_list(self.gen_exclude,
                                           "generic",
                                           dangerous=False)
                if self.share_exclude:
                    self._create_test_list(self.share_exclude,
                                           "shared",
                                           dangerous=False)
        if self.detected_distro.name is not 'SuSE':
            if process.system('useradd 123456-fsgqa',
                              sudo=True,
                              ignore_status=True):
                self.log.warn('useradd 123456-fsgqa failed')
            if process.system('useradd fsgqa', sudo=True, ignore_status=True):
                self.log.warn('useradd fsgqa failed')
        else:
            if process.system('useradd -m -U fsgqa',
                              sudo=True,
                              ignore_status=True):
                self.log.warn('useradd fsgqa failed')
            if process.system('groupadd sys', sudo=True, ignore_status=True):
                self.log.warn('groupadd sys failed')
        if not os.path.exists(self.scratch_mnt):
            os.makedirs(self.scratch_mnt)
        if not os.path.exists(self.test_mnt):
            os.makedirs(self.test_mnt)
    def setUp(self):
        """
        Build xfstest
        Source: git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git
        """
        self.use_dd = False
        root_fs = process.system_output("df -T / | awk 'END {print $2}'",
                                        shell=True).decode("utf-8")
        if root_fs in ['ext3', 'ext4']:
            self.use_dd = True
        self.dev_type = self.params.get('type', default='loop')

        sm = SoftwareManager()

        self.detected_distro = distro.detect()

        packages = [
            'e2fsprogs', 'automake', 'gcc', 'quota', 'attr', 'make',
            'xfsprogs', 'gawk'
        ]
        if self.detected_distro.name in ['Ubuntu', 'debian']:
            packages.extend([
                'xfslibs-dev', 'uuid-dev', 'libuuid1', 'libattr1-dev',
                'libacl1-dev', 'libgdbm-dev', 'uuid-runtime', 'libaio-dev',
                'fio', 'dbench'
            ])
            if self.detected_distro.version in ['14']:
                packages.extend(['libtool'])
            elif self.detected_distro.version in ['18', '20']:
                packages.extend(['libtool-bin', 'libgdbm-compat-dev'])
            else:
                packages.extend(['libtool-bin'])

        elif self.detected_distro.name in ['centos', 'fedora', 'rhel', 'SuSE']:
            if self.dev_type == 'nvdimm':
                packages.extend(['ndctl', 'parted'])
                if self.detected_distro.name == 'rhel':
                    packages.extend(['daxctl'])
            packages.extend([
                'acl', 'bc', 'dump', 'indent', 'libtool', 'lvm2', 'xfsdump',
                'psmisc', 'sed', 'libacl-devel', 'libattr-devel',
                'libaio-devel', 'libuuid-devel', 'openssl-devel',
                'xfsprogs-devel'
            ])

            if self.detected_distro.name == 'SuSE':
                packages.extend(['libbtrfs-devel', 'libcap-progs'])
            else:
                packages.extend(['btrfs-progs-devel'])

            packages_remove = ['indent', 'btrfs-progs-devel']
            if self.detected_distro.name == 'rhel' and\
                    self.detected_distro.version.startswith('8'):
                packages = list(set(packages) - set(packages_remove))
            elif self.detected_distro.name == 'rhel' and\
                    self.detected_distro.version.startswith('9'):
                packages_remove.extend(['dump'])
                packages = list(set(packages) - set(packages_remove))

            if self.detected_distro.name in ['centos', 'fedora']:
                packages.extend(['fio', 'dbench'])
        else:
            self.cancel("test not supported in %s" % self.detected_distro.name)

        for package in packages:
            if not sm.check_installed(package) and not sm.install(package):
                self.cancel("Fail to install %s required for this test." %
                            package)
        self.skip_dangerous = self.params.get('skip_dangerous', default=True)
        self.test_range = self.params.get('test_range', default=None)
        self.scratch_mnt = self.params.get('scratch_mnt',
                                           default='/mnt/scratch')
        self.test_mnt = self.params.get('test_mnt', default='/mnt/test')
        self.disk_mnt = self.params.get('disk_mnt', default='/mnt/loop_device')
        self.fs_to_test = self.params.get('fs', default='ext4')

        if process.system('which mkfs.%s' % self.fs_to_test,
                          ignore_status=True):
            self.cancel('Unknown filesystem %s' % self.fs_to_test)
        mount = True
        self.devices = []
        self.log_devices = []
        shutil.copyfile(self.get_data('local.config'),
                        os.path.join(self.teststmpdir, 'local.config'))
        shutil.copyfile(self.get_data('group'),
                        os.path.join(self.teststmpdir, 'group'))

        self.log_test = self.params.get('log_test', default='')
        self.log_scratch = self.params.get('log_scratch', default='')

        if self.dev_type == 'loop':
            base_disk = self.params.get('disk', default=None)
            loop_size = self.params.get('loop_size', default='7GiB')
            if not base_disk:
                # Using root for file creation by default
                check = (int(loop_size.split('GiB')[0]) * 2) + 1
                if disk.freespace('/') / 1073741824 > check:
                    self.disk_mnt = ''
                    mount = False
                else:
                    self.cancel('Need %s GB to create loop devices' % check)
            self._create_loop_device(base_disk, loop_size, mount)
        elif self.dev_type == 'nvdimm':
            self.setup_nvdimm()
        else:
            self.test_dev = self.params.get('disk_test', default=None)
            self.scratch_dev = self.params.get('disk_scratch', default=None)
            self.devices.extend([self.test_dev, self.scratch_dev])
        # mkfs for devices
        if self.devices:
            cfg_file = os.path.join(self.teststmpdir, 'local.config')
            self.mkfs_opt = self.params.get('mkfs_opt', default='')
            self.mount_opt = self.params.get('mount_opt', default='')
            with open(cfg_file, "r") as sources:
                lines = sources.readlines()
            with open(cfg_file, "w") as sources:
                for line in lines:
                    if line.startswith('export TEST_DEV'):
                        sources.write(
                            re.sub(r'export TEST_DEV=.*',
                                   'export TEST_DEV=%s' % self.devices[0],
                                   line))
                    elif line.startswith('export TEST_DIR'):
                        sources.write(
                            re.sub(r'export TEST_DIR=.*',
                                   'export TEST_DIR=%s' % self.test_mnt, line))
                    elif line.startswith('export SCRATCH_DEV'):
                        sources.write(
                            re.sub(r'export SCRATCH_DEV=.*',
                                   'export SCRATCH_DEV=%s' % self.devices[1],
                                   line))
                    elif line.startswith('export SCRATCH_MNT'):
                        sources.write(
                            re.sub(r'export SCRATCH_MNT=.*',
                                   'export SCRATCH_MNT=%s' % self.scratch_mnt,
                                   line))
                        break
            with open(cfg_file, "a") as sources:
                if self.log_test:
                    sources.write('export USE_EXTERNAL=yes\n')
                    sources.write('export TEST_LOGDEV="%s"\n' % self.log_test)
                    self.log_devices.append(self.log_test)
                if self.log_scratch:
                    sources.write('export SCRATCH_LOGDEV="%s"\n' %
                                  self.log_scratch)
                    self.log_devices.append(self.log_scratch)
                if self.mkfs_opt:
                    sources.write('MKFS_OPTIONS="%s"\n' % self.mkfs_opt)
                if self.mount_opt:
                    sources.write('MOUNT_OPTIONS="%s"\n' % self.mount_opt)
            self.logdev_opt = self.params.get('logdev_opt', default='')
            for dev in self.log_devices:
                dev_obj = partition.Partition(dev)
                dev_obj.mkfs(fstype=self.fs_to_test, args=self.mkfs_opt)
            for ite, dev in enumerate(self.devices):
                dev_obj = partition.Partition(dev)
                if self.logdev_opt:
                    dev_obj.mkfs(fstype=self.fs_to_test,
                                 args='%s %s=%s' %
                                 (self.mkfs_opt, self.logdev_opt,
                                  self.log_devices[ite]))
                else:
                    dev_obj.mkfs(fstype=self.fs_to_test, args=self.mkfs_opt)

        git.get_repo('git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git',
                     destination_dir=self.teststmpdir)

        build.make(self.teststmpdir)
        self.available_tests = self._get_available_tests()

        self.test_list = self._create_test_list(self.test_range)
        self.log.info("Tests available in srcdir: %s",
                      ", ".join(self.available_tests))
        if not self.test_range:
            self.exclude = self.params.get('exclude', default=None)
            self.gen_exclude = self.params.get('gen_exclude', default=None)
            self.share_exclude = self.params.get('share_exclude', default=None)
            if self.exclude or self.gen_exclude or self.share_exclude:
                self.exclude_file = os.path.join(self.teststmpdir, 'exclude')
                if self.exclude:
                    self._create_test_list(self.exclude,
                                           self.fs_to_test,
                                           dangerous=False)
                if self.gen_exclude:
                    self._create_test_list(self.gen_exclude,
                                           "generic",
                                           dangerous=False)
                if self.share_exclude:
                    self._create_test_list(self.share_exclude,
                                           "shared",
                                           dangerous=False)
        if self.detected_distro.name is not 'SuSE':
            if process.system('useradd 123456-fsgqa',
                              sudo=True,
                              ignore_status=True):
                self.log.warn('useradd 123456-fsgqa failed')
            if process.system('useradd fsgqa', sudo=True, ignore_status=True):
                self.log.warn('useradd fsgqa failed')
        else:
            if process.system('useradd -m -U fsgqa',
                              sudo=True,
                              ignore_status=True):
                self.log.warn('useradd fsgqa failed')
            if process.system('groupadd sys', sudo=True, ignore_status=True):
                self.log.warn('groupadd sys failed')
        if not os.path.exists(self.scratch_mnt):
            os.makedirs(self.scratch_mnt)
        if not os.path.exists(self.test_mnt):
            os.makedirs(self.test_mnt)
Example #23
0
    def setUp(self):
        """
        Build xfstest
        Source: git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git
        """
        self.use_dd = False
        root_fs = process.system_output("df -T / | awk 'END {print $2}'",
                                        shell=True)
        if root_fs in ['ext3', 'ext4']:
            self.use_dd = True
        sm = SoftwareManager()

        self.detected_distro = distro.detect()

        packages = [
            'e2fsprogs', 'automake', 'gcc', 'quota', 'attr', 'make',
            'xfsprogs', 'gawk'
        ]
        if 'Ubuntu' in self.detected_distro.name:
            packages.extend([
                'xfslibs-dev', 'uuid-dev', 'libuuid1', 'libattr1-dev',
                'libacl1-dev', 'libgdbm-dev', 'uuid-runtime', 'libaio-dev',
                'fio', 'dbench', 'btrfs-tools'
            ])
            if '14' in self.detected_distro.version:
                packages.extend(['libtool'])
            elif '18' in self.detected_distro.version:
                packages.extend(['libtool-bin', 'libgdbm-compat-dev'])
            else:
                packages.extend(['libtool-bin'])

        # FIXME: "redhat" as the distro name for RHEL is deprecated
        # on Avocado versions >= 50.0.  This is a temporary compatibility
        # enabler for older runners, but should be removed soon
        elif self.detected_distro.name in [
                'centos', 'fedora', 'rhel', 'redhat', 'SuSE'
        ]:
            packages.extend([
                'acl', 'bc', 'dump', 'indent', 'libtool', 'lvm2', 'xfsdump',
                'psmisc', 'sed', 'libacl-devel', 'libattr-devel',
                'libaio-devel', 'libuuid-devel', 'openssl-devel',
                'xfsprogs-devel'
            ])
            if self.detected_distro.name == 'SuSE':
                packages.extend(['libbtrfs-devel', 'libcap-progs'])
            else:
                packages.extend(['btrfs-progs-devel'])

            if self.detected_distro.name in ['centos', 'fedora']:
                packages.extend(['fio', 'dbench'])
        else:
            self.cancel("test not supported in %s" % self.detected_distro.name)

        for package in packages:
            if not sm.check_installed(package) and not sm.install(package):
                self.cancel("Fail to install %s required for this test." %
                            package)

        self.skip_dangerous = self.params.get('skip_dangerous', default=True)
        self.test_range = self.params.get('test_range', default=None)
        self.scratch_mnt = self.params.get('scratch_mnt',
                                           default='/mnt/scratch')
        self.test_mnt = self.params.get('test_mnt', default='/mnt/test')
        self.disk_mnt = self.params.get('disk_mnt', default='/mnt/loop_device')
        self.dev_type = self.params.get('type', default='loop')
        self.fs_to_test = self.params.get('fs', default='ext4')
        if process.system('which mkfs.%s' % self.fs_to_test,
                          ignore_status=True):
            self.cancel('Unknown filesystem %s' % self.fs_to_test)
        mount = True
        self.devices = []
        shutil.copyfile(self.get_data('local.config'),
                        os.path.join(self.teststmpdir, 'local.config'))
        shutil.copyfile(self.get_data('group'),
                        os.path.join(self.teststmpdir, 'group'))

        if self.dev_type == 'loop':
            base_disk = self.params.get('disk', default=None)
            loop_size = self.params.get('loop_size', default='7GiB')
            if not base_disk:
                # Using root for file creation by default
                check = (int(loop_size.split('GiB')[0]) * 2) + 1
                if disk.freespace('/') / 1073741824 > check:
                    self.disk_mnt = ''
                    mount = False
                else:
                    self.cancel('Need %s GB to create loop devices' % check)
            self._create_loop_device(base_disk, loop_size, mount)
        else:
            self.test_dev = self.params.get('disk_test', default=None)
            self.scratch_dev = self.params.get('disk_scratch', default=None)
            self.devices.extend([self.test_dev, self.scratch_dev])
        # mkfs for devices
        if self.devices:
            cfg_file = os.path.join(self.teststmpdir, 'local.config')
            with open(cfg_file, "r") as sources:
                lines = sources.readlines()
            with open(cfg_file, "w") as sources:
                for line in lines:
                    if line.startswith('export TEST_DEV'):
                        sources.write(
                            re.sub(r'export TEST_DEV=.*',
                                   'export TEST_DEV=%s' % self.devices[0],
                                   line))
                    elif line.startswith('export TEST_DIR'):
                        sources.write(
                            re.sub(r'export TEST_DIR=.*',
                                   'export TEST_DIR=%s' % self.test_mnt, line))
                    elif line.startswith('export SCRATCH_DEV'):
                        sources.write(
                            re.sub(r'export SCRATCH_DEV=.*',
                                   'export SCRATCH_DEV=%s' % self.devices[1],
                                   line))
                    elif line.startswith('export SCRATCH_MNT'):
                        sources.write(
                            re.sub(r'export SCRATCH_MNT=.*',
                                   'export SCRATCH_MNT=%s' % self.scratch_mnt,
                                   line))
                        break

            for dev in self.devices:
                dev_obj = partition.Partition(dev)
                dev_obj.mkfs(fstype=self.fs_to_test)

        git.get_repo('git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git',
                     destination_dir=self.teststmpdir)

        build.make(self.teststmpdir)
        self.available_tests = self._get_available_tests()

        self.test_list = self._create_test_list(self.test_range)
        self.log.info("Tests available in srcdir: %s",
                      ", ".join(self.available_tests))
        if not self.test_range:
            self.exclude = self.params.get('exclude', default=None)
            self.gen_exclude = self.params.get('gen_exclude', default=None)
            self.share_exclude = self.params.get('share_exclude', default=None)
            if self.exclude or self.gen_exclude or self.share_exclude:
                self.exclude_file = os.path.join(self.teststmpdir, 'exclude')
                if self.exclude:
                    self._create_test_list(self.exclude,
                                           self.fs_to_test,
                                           dangerous=False)
                if self.gen_exclude:
                    self._create_test_list(self.gen_exclude,
                                           "generic",
                                           dangerous=False)
                if self.share_exclude:
                    self._create_test_list(self.share_exclude,
                                           "shared",
                                           dangerous=False)
        if self.detected_distro.name is not 'SuSE':
            process.run('useradd 123456-fsgqa', sudo=True)
            process.run('useradd fsgqa', sudo=True)
        else:
            process.run('useradd -m -U fsgqa', sudo=True)
            process.run('groupadd sys', sudo=True)
        if not os.path.exists(self.scratch_mnt):
            os.makedirs(self.scratch_mnt)
        if not os.path.exists(self.test_mnt):
            os.makedirs(self.test_mnt)
    def setup_nvdimm(self):
        self.logflag = self.params.get('logdev', default=False)
        self.plib = pmem.PMem()
        self.plib.enable_region()
        regions = sorted(self.plib.run_ndctl_list('-R'),
                         key=lambda i: i['size'],
                         reverse=True)
        if not regions:
            self.cancel("Nvdimm test with no region support")

        self.region = self.plib.run_ndctl_list_val(regions[0], 'dev')
        if self.plib.is_region_legacy(self.region):
            if not len(regions) > 1:
                self.cancel("Not supported with single legacy region")
            if self.logflag:
                self.log.info("Using loop devices as log devices")
                check = 2
                mount = True
                if disk.freespace('/') / 1073741824 > check:
                    self.disk_mnt = ''
                    mount = False
                else:
                    self.cancel('Need %s GB to create loop devices' % check)
                self._create_loop_device(None, '2038M', mount)
                self.log_test = self.devices.pop()
                self.log_scratch = self.devices.pop()
            namespaces = self.plib.run_ndctl_list('-N -r %s' % self.region)
            pmem_dev = self.plib.run_ndctl_list_val(namespaces[0], 'blockdev')
            self.test_dev = "/dev/%s" % pmem_dev
            region_2 = self.plib.run_ndctl_list_val(regions[1], 'dev')
            namespaces = self.plib.run_ndctl_list('-N -r %s' % region_2)
            pmem_dev = self.plib.run_ndctl_list_val(namespaces[0], 'blockdev')
            self.scratch_dev = "/dev/%s" % pmem_dev
            self.devices.extend([self.test_dev, self.scratch_dev])
        else:
            if self.logflag:
                if not len(regions) > 1:
                    self.log.info('Using 10% space of device for logdev')
                    self.region_ldev = self.region
                    region_size = self.plib.run_ndctl_list_val(
                        self.plib.run_ndctl_list('-r %s' %
                                                 self.region_ldev)[0], 'size')
                    logdev_size = int(region_size * 0.10)
                    dev_size = region_size - logdev_size
                    size_align = self.get_size_alignval()
                    dev_size = dev_size // 2
                    dev_size = (dev_size // size_align) * size_align
                    logdev_size = logdev_size // 2
                    logdev_size = (logdev_size // size_align) * size_align
                else:
                    dev_size = self.get_half_region_size(self.region)
                    self.region_ldev = self.plib.run_ndctl_list_val(
                        regions[1], 'dev')
                    logdev_size = self.get_half_region_size(
                        region=self.region_ldev)
                    self.plib.destroy_namespace(region=self.region, force=True)
                self.plib.destroy_namespace(region=self.region_ldev,
                                            force=True)
                # XFS restrict max log size to 2136997888, which is 10M less
                # than 2GB, not 16M page-aligned, hence rounding-off to nearest
                # 16M align value 2130706432, which is 16M less than 2GiB
                logdev_size = min(logdev_size, 2130706432)
                # log device to be created in sector mode
                self.plib.create_namespace(region=self.region_ldev,
                                           mode='sector',
                                           sector_size='512',
                                           size=logdev_size)
                self.plib.create_namespace(region=self.region_ldev,
                                           mode='sector',
                                           sector_size='512',
                                           size=logdev_size)

                namespaces = self.plib.run_ndctl_list('-N -r %s -m sector' %
                                                      self.region_ldev)
                log_dev = self.plib.run_ndctl_list_val(namespaces[0],
                                                       'blockdev')
                self.log_test = "/dev/%s" % log_dev
                log_dev = self.plib.run_ndctl_list_val(namespaces[1],
                                                       'blockdev')
                self.log_scratch = "/dev/%s" % log_dev
            else:
                self.plib.destroy_namespace(region=self.region, force=True)
                dev_size = self.get_half_region_size(self.region)
                self.log_test = None
                self.log_scratch = None
            self.plib.create_namespace(region=self.region, size=dev_size)
            self.plib.create_namespace(region=self.region, size=dev_size)
            namespaces = self.plib.run_ndctl_list('-N -r %s -m fsdax' %
                                                  self.region)
            pmem_dev = self.plib.run_ndctl_list_val(namespaces[0], 'blockdev')
            self.test_dev = "/dev/%s" % pmem_dev
            pmem_dev = self.plib.run_ndctl_list_val(namespaces[1], 'blockdev')
            self.scratch_dev = "/dev/%s" % pmem_dev
            self.devices.extend([self.test_dev, self.scratch_dev])
Example #25
0
    def setUp(self):
        """
        Build xfstest
        Source: git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git
        """
        if process.system_output("df -T / | awk 'END {print $2}'",
                                 shell=True) == 'ext3':
            self.skip('Test does not support ext3 root file system')
        sm = SoftwareManager()

        self.detected_distro = distro.detect()

        packages = [
            'e2fsprogs', 'automake', 'gcc', 'quota', 'attr', 'make',
            'xfsprogs', 'gawk'
        ]
        if 'Ubuntu' in self.detected_distro.name:
            packages.extend([
                'xfslibs-dev', 'uuid-dev', 'libtool-bin', 'libuuid1',
                'libattr1-dev', 'libacl1-dev', 'libgdbm-dev', 'uuid-runtime',
                'libaio-dev', 'fio', 'dbench'
            ])

        # FIXME: "redhat" as the distro name for RHEL is deprecated
        # on Avocado versions >= 50.0.  This is a temporary compatibility
        # enabler for older runners, but should be removed soon
        elif self.detected_distro.name in [
                'centos', 'fedora', 'rhel', 'redhat', 'SuSE'
        ]:
            packages.extend([
                'acl', 'bc', 'dump', 'indent', 'libtool', 'lvm2', 'xfsdump',
                'psmisc', 'sed', 'libacl-devel', 'libattr-devel',
                'libaio-devel', 'libuuid-devel', 'openssl-devel',
                'xfsprogs-devel'
            ])
            if self.detected_distro.name == 'SuSE':
                packages.extend(['libbtrfs-devel'])
            else:
                packages.extend(['btrfs-progs-devel'])

            if self.detected_distro.name in ['centos', 'fedora']:
                packages.extend(['fio', 'dbench'])
        else:
            self.skip("test not supported in %s" % self.detected_distro.name)

        for package in packages:
            if not sm.check_installed(package) and not sm.install(package):
                self.skip("Fail to install %s required for this test." %
                          package)

        self.skip_dangerous = self.params.get('skip_dangerous', default=True)
        self.test_range = self.params.get('test_range', default=None)
        self.scratch_mnt = self.params.get('scratch_mnt',
                                           default='/mnt/scratch')
        self.test_mnt = self.params.get('test_mnt', default='/mnt/test')
        self.disk_mnt = self.params.get('disk_mnt', default='/mnt/loop_device')
        self.dev_type = self.params.get('type', default='loop')
        self.fs_to_test = self.params.get('fs', default='ext4')
        if process.system('which mkfs.%s' % self.fs_to_test,
                          ignore_status=True):
            self.skip('Unknown filesystem %s' % self.fs_to_test)
        mount = True
        self.devices = []
        shutil.copyfile(os.path.join(self.datadir, 'local.config'),
                        os.path.join(self.srcdir, 'local.config'))
        shutil.copyfile(os.path.join(self.datadir, 'group'),
                        os.path.join(self.srcdir, 'group'))

        if self.dev_type == 'loop':
            base_disk = self.params.get('disk', default=None)
            loop_size = self.params.get('loop_size', default='9GiB')
            if not base_disk:
                # Using root for file creation by default
                if disk.freespace('/') / 1073741824 > 15:
                    self.disk_mnt = ''
                    mount = False
                else:
                    self.skip('Need 15 GB to create loop devices')
            self._create_loop_device(base_disk, loop_size, mount)
        else:
            self.test_dev = self.params.get('disk_test', default=None)
            self.scratch_dev = self.params.get('disk_scratch', default=None)
            self.devices.extend([self.test_dev, self.scratch_dev])
        # mkfs for devices
        if self.devices:
            line = ('export TEST_DEV=%s' % self.devices[0]).replace('/', '\/')
            process.system('sed -i "s/export TEST_DEV=.*/%s/g" %s' %
                           (line, os.path.join(self.srcdir, 'local.config')),
                           shell=True)
            line = ('export SCRATCH_DEV=%s' % self.devices[1]).replace(
                '/', '\/')
            process.system('sed -i "s/export SCRATCH_DEV=.*/%s/g" %s' %
                           (line, os.path.join(self.srcdir, 'local.config')),
                           shell=True)
            for dev in self.devices:
                dev_obj = partition.Partition(dev)
                dev_obj.mkfs(fstype=self.fs_to_test)

        git.get_repo('git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git',
                     destination_dir=self.srcdir)

        build.make(self.srcdir)
        self.available_tests = self._get_available_tests()

        self.test_list = self._create_test_list()
        self.log.info("Tests available in srcdir: %s",
                      ", ".join(self.available_tests))
        process.run('useradd fsgqa', sudo=True)
        if self.detected_distro.name is not 'SuSE':
            process.run('useradd 123456-fsgqa', sudo=True)
        if not os.path.exists(self.scratch_mnt):
            os.makedirs(self.scratch_mnt)
        if not os.path.exists(self.test_mnt):
            os.makedirs(self.test_mnt)