def setUp(self): """ Setup scripts/disks to memory hog """ smm = SoftwareManager() self.memsize = self.params.get('memory_size', default=None) if not self.memsize: self.memsize = int(memory.meminfo.MemFree.b * 0.9) self.file_type = self.params.get('file_type', default=None) deps = ['gcc'] detected_distro = distro.detect() if detected_distro.name in ["Ubuntu", "debian"]: deps.extend(['libnuma-dev']) elif detected_distro.name in ["centos", "rhel", "fedora"]: deps.extend(['numactl-devel']) else: deps.extend(['libnuma-devel']) if self.file_type: if self.file_type == 'nvdimm': deps.extend(['ndctl']) if detected_distro.name == "rhel": deps.extend(['daxctl']) for package in deps: if not smm.check_installed(package) and not smm.install(package): self.cancel('%s is needed for the test to be run' % package) srcdir = os.path.join(self.workdir, 'memhog') if not os.path.exists(srcdir): os.makedirs(srcdir) for fname in ['memhog.c', 'util.c', 'util.h']: fetch_file = self.fetch_asset( fname, locations=[ 'https://raw.githubusercontent.com/numactl/' 'numactl/master/%s' % fname ], expire='7d') shutil.copyfile(fetch_file, os.path.join(srcdir, fname)) if self.file_type: if self.file_type == 'nvdimm': self.back_file = self.setup_nvdimm() disk_size = int(disk.freespace(self.mnt_dir) * 0.95) else: self.back_file = '/tmp/back_file' disk_size = int(disk.freespace('/tmp') * 0.95) if self.memsize > disk_size: self.memsize = disk_size cmd = 'fallocate -l %s %s' % (self.memsize, self.back_file) if process.system(cmd, ignore_status=True): self.cancel('Could not create file to be backed') os.chdir(srcdir) if process.system('gcc -lnuma -o memhog memhog.c util.c'): self.cancel('Compiling source failed')
def setUp(self): """ Downloads a copy of the linux kernel, calculate an estimated size of the uncompressed tarball, use this value to calculate the number of copies of the linux kernel that will be uncompressed. """ self.nfail = 0 base_dir = self.teststmpdir if disk.freespace("/home") > disk.freespace(self.teststmpdir): base_dir = "/home" self.log.info("Using %s as base dir for the test" % base_dir) self.tmpdir = tempfile.mkdtemp(dir=base_dir) self.tmpdir = self.params.get('dir_to_extract', default=self.tmpdir) self.base_dir = os.path.join(self.tmpdir, 'linux.orig') tarball_base = self.params.get('tarball_base', default='linux-2.6.18.tar.bz2') kernel_repo = self.params.get('kernel_repo', default='http://www.kernel.org/pub/' 'linux/kernel/v2.6') tarball_url = os.path.join(kernel_repo, tarball_base) tarball_md5 = self.params.get('tarball_md5', default='296a6d150d260144639c3664d127d1' '74') parallel = self.params.get('parallel', default=True) self.parallel = parallel self.log.info('Downloading linux kernel tarball') self.tarball = self.fetch_asset(tarball_url, asset_hash=tarball_md5, algorithm='md5') size_tarball = os.path.getsize(self.tarball) // 1024 // 1024 # Estimation of the tarball size after uncompression os.chdir(self.tmpdir) # This is the reference copy of the linux tarball # that will be used for subsequent comparisons self.log.info('Unpacking base copy') archive.extract(self.tarball, self.base_dir) est_size = int( process.system_output( 'du -sb %s' % self.base_dir).split()[0].decode()) // 1048576 self.sim_cps = self.get_sim_cps(est_size) self.log.info('Source file: %s', tarball_base) self.log.info('Megabytes per copy: %s', size_tarball) self.log.info('Estimated size after uncompression: %s', est_size) self.log.info('Number of copies: %s', self.sim_cps) self.log.info('Parallel: %s', parallel) # Verify if space is available in disk disk_free_mb = (disk.freespace(self.tmpdir) // 1024) // 1024 if disk_free_mb < (est_size * self.sim_cps): self.cancel("Space not available to extract the %s linux tars\n" "Mount and Use other partitions in dir_to_extract arg " "to run the test" % self.sim_cps)
def setUp(self): self.iter = int(self.params.get('iterations', default='5')) self.memsize = int(self.params.get( 'mem_size', default=memory.memtotal() * 0.9)) self.ddfile = os.path.join(self.workdir, 'ddfile') if (disk.freespace(self.workdir) / 1024) < self.memsize: self.cancel('%sM is needed for the test to be run' % self.memsize)
def setUp(self): ''' Build interbench Source: http://ck.kolivas.org/apps/interbench/interbench-0.31.tar.bz2 ''' sm_manager = SoftwareManager() for pkg in ['gcc', 'patch']: if (not sm_manager.check_installed(pkg) and not sm_manager.install(pkg)): self.cancel("%s is needed for the test to be run" % pkg) disk_free_mb = (disk.freespace(self.teststmpdir) / 1024) / 1024 if memory.memtotal() / 1024 > disk_free_mb: self.cancel('Disk space is less than total memory. Skipping test') tarball = self.fetch_asset('http://slackware.cs.utah.edu/pub/kernel' '.org/pub/linux/kernel/people/ck/apps/' 'interbench/interbench-0.31.tar.gz') data_dir = os.path.abspath(self.datadir) archive.extract(tarball, self.srcdir) version = os.path.basename(tarball.split('.tar.')[0]) self.srcdir = os.path.join(self.srcdir, version) # Patch for make file os.chdir(self.srcdir) makefile_patch = 'patch -p1 < %s ' % (os.path.join( data_dir, 'makefile_fix.patch')) process.run(makefile_patch, shell=True) build.make(self.srcdir)
def setUp(self): ''' Build interbench Source: http://ck.kolivas.org/apps/interbench/interbench-0.31.tar.bz2 ''' sm_manager = SoftwareManager() for pkg in ['gcc', 'patch']: if (not sm_manager.check_installed(pkg) and not sm_manager.install(pkg)): self.cancel("%s is needed for the test to be run" % pkg) disk_free_mb = (disk.freespace(self.teststmpdir) / 1024) / 1024 if memory.memtotal()/1024 > disk_free_mb: self.cancel('Disk space is less than total memory. Skipping test') tarball = self.fetch_asset('http://slackware.cs.utah.edu/pub/kernel' '.org/pub/linux/kernel/people/ck/apps/' 'interbench/interbench-0.31.tar.gz') data_dir = os.path.abspath(self.datadir) archive.extract(tarball, self.srcdir) version = os.path.basename(tarball.split('.tar.')[0]) self.sourcedir = os.path.join(self.srcdir, version) # Patch for make file os.chdir(self.sourcedir) makefile_patch = 'patch -p1 < %s ' % ( os.path.join(data_dir, 'makefile_fix.patch')) process.run(makefile_patch, shell=True) build.make(self.sourcedir)
def setUp(self): ''' Build interbench Source: http://ck.kolivas.org/apps/interbench/interbench-0.31.tar.bz2 ''' sm_manager = SoftwareManager() for pkg in ['gcc', 'patch']: if (not sm_manager.check_installed(pkg) and not sm_manager.install(pkg)): self.cancel("%s is needed for the test to be run" % pkg) disk_free_b = disk.freespace(self.teststmpdir) if memory.meminfo.MemTotal.b > disk_free_b: self.cancel('Disk space is less than total memory. Skipping test') tarball = self.fetch_asset( 'http://ck.kolivas.org/apps/interbench/interbench-0.31.tar.bz2') archive.extract(tarball, self.workdir) version = os.path.basename(tarball.split('.tar.')[0]) self.sourcedir = os.path.join(self.workdir, version) # Patch for make file os.chdir(self.sourcedir) makefile_patch = 'patch -p1 < %s ' % self.get_data( 'makefile_fix.patch') process.run(makefile_patch, shell=True) build.make(self.sourcedir)
def setUp(self): self.iter = int(self.params.get('iterations', default='5')) dir_to_use = self.params.get('dir_to_use', default=None) self.memsize = int(self.params.get( 'mem_size', default=memory.meminfo.MemFree.k * 0.9)) if not dir_to_use: base_dir = self.teststmpdir if disk.freespace("/home") > disk.freespace(self.teststmpdir): base_dir = "/home" else: base_dir = dir_to_use self.log.info("Using %s as base dir for the test" % base_dir) self.tmpdir = tempfile.mkdtemp(dir=base_dir) self.ddfile = os.path.join(self.tmpdir, 'ddfile') if (disk.freespace(self.tmpdir) // 1024) < self.memsize: self.cancel('%sM is needed for the test to be run' % (self.memsize // 1024))
def test(self): """ Execute 'stress' with proper arguments. """ length = self.params.get('stress_lenght', default=60) threads = self.params.get('threads', default=None) memory_per_thread = self.params.get('memory_per_thread', default=None) file_size_per_thread = self.params.get('file_size_per_thread', default=None) if threads is None: # We will use 2 workers of each type for each CPU detected threads = 2 * multiprocessing.cpu_count() if memory_per_thread is None: # Sometimes the default memory used by each memory worker (256 M) # might make our machine go OOM and then funny things might start # to happen. Let's avoid that. mb = (memory.freememtotal() + memory.read_from_meminfo('SwapFree') / 2) memory_per_thread = (mb * 1024) / threads if file_size_per_thread is None: # Even though unlikely, it's good to prevent from allocating more # disk than this machine actually has on its autotest directory # (limit the amount of disk used to max of 90 % of free space) free_disk = disk.freespace(self.sourcedir) file_size_per_thread = 1024**2 if (0.9 * free_disk) < file_size_per_thread * threads: file_size_per_thread = (0.9 * free_disk) / threads # Number of CPU workers spinning on sqrt() args = '--cpu %d ' % threads # Number of IO workers spinning on sync() args += '--io %d ' % threads # Number of Memory workers spinning on malloc()/free() args += '--vm %d ' % threads # Amount of memory used per each worker args += '--vm-bytes %d ' % memory_per_thread # Number of HD workers spinning on write()/ulink() args += '--hdd %d ' % threads # Size of the files created by each worker in bytes args += '--hdd-bytes %d ' % file_size_per_thread # Time for which the stress test will run args += '--timeout %d ' % length # Verbose flag args += '--verbose' os.chdir(self.sourcedir) cmd = ('./src/stress %s' % args) process.run(cmd)
def test(self): """ Execute 'stress' with proper arguments. """ length = self.params.get('stress_lenght', default=60) threads = self.params.get('threads', default=None) memory_per_thread = self.params.get('memory_per_thread', default=None) file_size_per_thread = self.params.get('file_size_per_thread', default=None) if threads is None: # We will use 2 workers of each type for each CPU detected threads = 2 * multiprocessing.cpu_count() if memory_per_thread is None: # Sometimes the default memory used by each memory worker (256 M) # might make our machine go OOM and then funny things might start # to happen. Let's avoid that. mb = (memory.freememtotal() + memory.read_from_meminfo('SwapFree') / 2) memory_per_thread = (mb * 1024) / threads if file_size_per_thread is None: # Even though unlikely, it's good to prevent from allocating more # disk than this machine actually has on its autotest directory # (limit the amount of disk used to max of 90 % of free space) free_disk = disk.freespace(self.sourcedir) file_size_per_thread = 1024 ** 2 if (0.9 * free_disk) < file_size_per_thread * threads: file_size_per_thread = (0.9 * free_disk) / threads # Number of CPU workers spinning on sqrt() args = '--cpu %d ' % threads # Number of IO workers spinning on sync() args += '--io %d ' % threads # Number of Memory workers spinning on malloc()/free() args += '--vm %d ' % threads # Amount of memory used per each worker args += '--vm-bytes %d ' % memory_per_thread # Number of HD workers spinning on write()/ulink() args += '--hdd %d ' % threads # Size of the files created by each worker in bytes args += '--hdd-bytes %d ' % file_size_per_thread # Time for which the stress test will run args += '--timeout %d ' % length # Verbose flag args += '--verbose' os.chdir(self.sourcedir) cmd = ('./src/stress %s' % args) process.run(cmd)
def setUp(self): ''' Sets required params for dd workload and mounts the tmpfs ''' # Get required mem info self.mem_path = os.path.join(data_dir.get_tmp_dir(), 'thp_space') self.block_size = int(mmap.PAGESIZE) / 1024 # add mount point os.mkdir(self.mem_path) self.device = Partition(device="none", mountpoint=self.mem_path) self.device.mount(mountpoint=self.mem_path, fstype="tmpfs") free_space = (disk.freespace(self.mem_path)) / 1024 # Leaving out some free space in tmpfs self.count = (free_space / self.block_size) - 3
def setUp(self): ''' Sets required params for dd workload and mounts the tmpfs ''' # Get required mem info self.mem_path = os.path.join(data_dir.get_tmp_dir(), 'thp_space') self.block_size = int(mmap.PAGESIZE) / 1024 # add mount point if os.path.exists(self.mem_path): os.makedirs(self.mem_path) self.device = Partition(device="none", mountpoint=self.mem_path) self.device.mount(mountpoint=self.mem_path, fstype="tmpfs") free_space = (disk.freespace(self.mem_path)) / 1024 # Leaving out some free space in tmpfs self.count = (free_space / self.block_size) - 3
def _init_params(self): """ Retrieves and checks the test params """ self.disk = self.params.get('disk', default=None) self.dirs = self.params.get('dir', default=self.workdir) self.fstype = self.params.get('fs', default='ext4') memory_mb = memory.memtotal() / 1024 self.chunk_mb = int(self.params.get('chunk_mb', default=None)) if self.chunk_mb is None: # By default total RAM self.chunk_mb = memory_mb if self.chunk_mb == 0: self.chunk_mb = 1 if memory_mb > self.chunk_mb: self.cancel("Chunk size has to be greater or equal to RAM size. " "(%s > %s)" % (self.chunk_mb, memory_mb)) gigabytes = int(self.params.get('gigabytes', default=None)) if gigabytes is None: free = 107374182400 # cap it at 100GB by default free = min(utils_disk.freespace(self.dirs) / 1073741824, free) gigabytes = free self.no_chunks = 1024 * gigabytes / self.chunk_mb if self.no_chunks == 0: self.cancel("Free disk space is lower than chunk size (%s, %s)" % (1024 * gigabytes, self.chunk_mb)) self.log.info("Test will use %s chunks %sMB each in %sMB RAM using %s " "GB of disk space on %s dirs (%s).", self.no_chunks, self.chunk_mb, memory_mb, self.no_chunks * self.chunk_mb, len(self.dirs), self.dirs) if self.disk is not None: self.part_obj = Partition(self.disk, mountpoint=self.dirs) self.log.info("Unmounting the disk/dir if it is already mounted") self.part_obj.unmount() self.log.info("creating %s fs on %s", self.fstype, self.disk) self.part_obj.mkfs(self.fstype) self.log.info("mounting %s on %s", self.disk, self.dirs) try: self.part_obj.mount() except PartitionError: self.fail("Mounting disk %s on directory %s failed" % (self.disk, self.dirs))
def setUp(self): """ Downloads a copy of the linux kernel, calculate an estimated size of the uncompressed tarball, use this value to calculate the number of copies of the linux kernel that will be uncompressed. """ self.nfail = 0 self.tmpdir = data_dir.get_tmp_dir() self.tmpdir = self.params.get('dir_to_extract', default=self.tmpdir) self.base_dir = os.path.join(self.tmpdir, 'linux.orig') tarball_base = self.params.get('tarball_base', default='linux-2.6.18.tar.bz2') kernel_repo = self.params.get('kernel_repo', default='http://www.kernel.org/pub/' 'linux/kernel/v2.6') tarball_url = os.path.join(kernel_repo, tarball_base) tarball_md5 = self.params.get('tarball_md5', default='296a6d150d260144639c3664d127d1' '74') parallel = self.params.get('parallel', default=True) self.parallel = parallel self.log.info('Downloading linux kernel tarball') self.tarball = self.fetch_asset(tarball_url, asset_hash=tarball_md5, algorithm='md5') size_tarball = os.path.getsize(self.tarball) // 1024 // 1024 # Estimation of the tarball size after uncompression compress_ratio = 5 est_size = size_tarball * compress_ratio self.sim_cps = self.get_sim_cps(est_size) self.log.info('Source file: %s', tarball_base) self.log.info('Megabytes per copy: %s', size_tarball) self.log.info('Compress ratio: %s', compress_ratio) self.log.info('Estimated size after uncompression: %s', est_size) self.log.info('Number of copies: %s', self.sim_cps) self.log.info('Parallel: %s', parallel) # Verify if space is available in disk disk_free_mb = (disk.freespace(self.tmpdir) // 1024) // 1024 if disk_free_mb < (est_size * self.sim_cps): self.cancel("Space not available to extract the %s linux tars\n" "Mount and Use other partitions in dir_to_extract arg " "to run the test" % self.sim_cps)
def setUp(self): """ Downloads a copy of the linux kernel, calculate an estimated size of the uncompressed tarball, use this value to calculate the number of copies of the linux kernel that will be uncompressed. """ self.nfail = 0 self.tmpdir = data_dir.get_tmp_dir() self.tmpdir = self.params.get('dir_to_extract', default=self.tmpdir) tarball_base = self.params.get('tarball_base', default='linux-2.6.18.tar.bz2') kernel_repo = self.params.get('kernel_repo', default='http://www.kernel.org/pub/' 'linux/kernel/v2.6') tarball_url = os.path.join(kernel_repo, tarball_base) tarball_md5 = self.params.get('tarball_md5', default='296a6d150d260144639c3664d127d1' '74') parallel = self.params.get('parallel', default=True) self.parallel = parallel self.log.info('Downloading linux kernel tarball') self.tarball = self.fetch_asset(tarball_url, asset_hash=tarball_md5, algorithm='md5') size_tarball = os.path.getsize(self.tarball) / 1024 / 1024 # Estimation of the tarball size after uncompression compress_ratio = 5 est_size = size_tarball * compress_ratio self.sim_cps = self.get_sim_cps(est_size) self.log.info('Source file: %s', tarball_base) self.log.info('Megabytes per copy: %s', size_tarball) self.log.info('Compress ratio: %s', compress_ratio) self.log.info('Estimated size after uncompression: %s', est_size) self.log.info('Number of copies: %s', self.sim_cps) self.log.info('Parallel: %s', parallel) # Verify if space is available in disk disk_free_mb = (disk.freespace(self.tmpdir) / 1024) / 1024 if (disk_free_mb < est_size * self.sim_cps): self.cancel("Space not available to extract the %s linux tars\n" "Mount and Use other partitions in dir_to_extract arg " "to run the test" % self.sim_cps)
def _init_params(self): """ Retrieves and checks the test params """ disks = self.params.get('disks', default=None) if disks is None: # Avocado does not accept lists in params.get() disks = [self.workdir] elif isinstance(disks, basestring): # Allow specifying disks as str disks = disks.split(',') # it's string pylint: disable=E1101 for disk in disks: # Disks have to be mounted dirs if not os.path.isdir(disk): os.makedirs(disk) self.disks = disks memory_mb = memory.memtotal() / 1024 self.chunk_mb = self.params.get('chunk_mb', default=None) if self.chunk_mb is None: # By default total RAM self.chunk_mb = memory_mb if self.chunk_mb == 0: self.chunk_mb = 1 if memory_mb > self.chunk_mb: self.skip("Chunk size has to be greater or equal to RAM size. " "(%s > %s)" % (self.chunk_mb, memory_mb)) gigabytes = self.params.get('gigabytes', default=None) if gigabytes is None: free = 100 # cap it at 100GB by default for disk in self.disks: free = min(utils_disk.freespace(disk) / 1073741824, free) gigabytes = free self.no_chunks = 1024 * gigabytes / self.chunk_mb if self.no_chunks == 0: self.skip("Free disk space is lower than chunk size (%s, %s)" % (1024 * gigabytes, self.chunk_mb)) self.log.info("Test will use %s chunks %sMB each in %sMB RAM using %s " "GB of disk space on %s disks (%s).", self.no_chunks, self.chunk_mb, memory_mb, self.no_chunks * self.chunk_mb, len(self.disks), self.disks)
def setUp(self): ''' Sets required params for dd workload and mounts the tmpfs ''' # Get required mem info self.mem_path = os.path.join(data_dir.get_tmp_dir(), 'thp_space') self.block_size = int(mmap.PAGESIZE) // 1024 # add mount point if os.path.exists(self.mem_path): os.makedirs(self.mem_path) self.device = Partition(device="none", mountpoint=self.mem_path) self.device.mount(mountpoint=self.mem_path, fstype="tmpfs", mnt_check=False) free_space = (disk.freespace(self.mem_path)) // 1024 # Reserving some memory (out of 100% memory reserving 25% memory) res_free_space = int(free_space / 4) free_space = free_space - res_free_space # Leaving out some free space in tmpfs self.count = (free_space // self.block_size) - 3
def setUp(self): """ Build xfstest Source: git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git """ self.use_dd = False root_fs = process.system_output( "df -T / | awk 'END {print $2}'", shell=True) if root_fs in ['ext3', 'ext4']: self.use_dd = True sm = SoftwareManager() self.detected_distro = distro.detect() packages = ['e2fsprogs', 'automake', 'gcc', 'quota', 'attr', 'make', 'xfsprogs', 'gawk'] if 'Ubuntu' in self.detected_distro.name: packages.extend( ['xfslibs-dev', 'uuid-dev', 'libuuid1', 'libattr1-dev', 'libacl1-dev', 'libgdbm-dev', 'uuid-runtime', 'libaio-dev', 'fio', 'dbench', 'btrfs-tools']) if '14' in self.detected_distro.version: packages.extend(['libtool']) elif '18' in self.detected_distro.version: packages.extend(['libtool-bin', 'libgdbm-compat-dev']) else: packages.extend(['libtool-bin']) # FIXME: "redhat" as the distro name for RHEL is deprecated # on Avocado versions >= 50.0. This is a temporary compatibility # enabler for older runners, but should be removed soon elif self.detected_distro.name in ['centos', 'fedora', 'rhel', 'redhat', 'SuSE']: packages.extend(['acl', 'bc', 'dump', 'indent', 'libtool', 'lvm2', 'xfsdump', 'psmisc', 'sed', 'libacl-devel', 'libattr-devel', 'libaio-devel', 'libuuid-devel', 'openssl-devel', 'xfsprogs-devel']) if self.detected_distro.name == 'SuSE': packages.extend(['libbtrfs-devel', 'libcap-progs']) else: packages.extend(['btrfs-progs-devel']) if self.detected_distro.name in ['centos', 'fedora']: packages.extend(['fio', 'dbench']) else: self.cancel("test not supported in %s" % self.detected_distro.name) for package in packages: if not sm.check_installed(package) and not sm.install(package): self.cancel("Fail to install %s required for this test." % package) self.skip_dangerous = self.params.get('skip_dangerous', default=True) self.test_range = self.params.get('test_range', default=None) self.scratch_mnt = self.params.get( 'scratch_mnt', default='/mnt/scratch') self.test_mnt = self.params.get('test_mnt', default='/mnt/test') self.disk_mnt = self.params.get('disk_mnt', default='/mnt/loop_device') self.dev_type = self.params.get('type', default='loop') self.fs_to_test = self.params.get('fs', default='ext4') if process.system('which mkfs.%s' % self.fs_to_test, ignore_status=True): self.cancel('Unknown filesystem %s' % self.fs_to_test) mount = True self.devices = [] shutil.copyfile(self.get_data('local.config'), os.path.join(self.teststmpdir, 'local.config')) shutil.copyfile(self.get_data('group'), os.path.join(self.teststmpdir, 'group')) if self.dev_type == 'loop': base_disk = self.params.get('disk', default=None) loop_size = self.params.get('loop_size', default='7GiB') if not base_disk: # Using root for file creation by default check = (int(loop_size.split('GiB')[0]) * 2) + 1 if disk.freespace('/') / 1073741824 > check: self.disk_mnt = '' mount = False else: self.cancel('Need %s GB to create loop devices' % check) self._create_loop_device(base_disk, loop_size, mount) else: self.test_dev = self.params.get('disk_test', default=None) self.scratch_dev = self.params.get('disk_scratch', default=None) self.devices.extend([self.test_dev, self.scratch_dev]) # mkfs for devices if self.devices: cfg_file = os.path.join(self.teststmpdir, 'local.config') with open(cfg_file, "r") as sources: lines = sources.readlines() with open(cfg_file, "w") as sources: for line in lines: if line.startswith('export TEST_DEV'): sources.write( re.sub(r'export TEST_DEV=.*', 'export TEST_DEV=%s' % self.devices[0], line)) elif line.startswith('export TEST_DIR'): sources.write( re.sub(r'export TEST_DIR=.*', 'export TEST_DIR=%s' % self.test_mnt, line)) elif line.startswith('export SCRATCH_DEV'): sources.write(re.sub( r'export SCRATCH_DEV=.*', 'export SCRATCH_DEV=%s' % self.devices[1], line)) elif line.startswith('export SCRATCH_MNT'): sources.write( re.sub( r'export SCRATCH_MNT=.*', 'export SCRATCH_MNT=%s' % self.scratch_mnt, line)) break for dev in self.devices: dev_obj = partition.Partition(dev) dev_obj.mkfs(fstype=self.fs_to_test) git.get_repo('git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git', destination_dir=self.teststmpdir) build.make(self.teststmpdir) self.available_tests = self._get_available_tests() self.test_list = self._create_test_list(self.test_range) self.log.info("Tests available in srcdir: %s", ", ".join(self.available_tests)) if not self.test_range: self.exclude = self.params.get('exclude', default=None) self.gen_exclude = self.params.get('gen_exclude', default=None) self.share_exclude = self.params.get('share_exclude', default=None) if self.exclude or self.gen_exclude or self.share_exclude: self.exclude_file = os.path.join(self.teststmpdir, 'exclude') if self.exclude: self._create_test_list(self.exclude, self.fs_to_test, dangerous=False) if self.gen_exclude: self._create_test_list(self.gen_exclude, "generic", dangerous=False) if self.share_exclude: self._create_test_list(self.share_exclude, "shared", dangerous=False) if self.detected_distro.name is not 'SuSE': process.run('useradd 123456-fsgqa', sudo=True) process.run('useradd fsgqa', sudo=True) else: process.run('useradd -m -U fsgqa', sudo=True) process.run('groupadd sys', sudo=True) if not os.path.exists(self.scratch_mnt): os.makedirs(self.scratch_mnt) if not os.path.exists(self.test_mnt): os.makedirs(self.test_mnt)
def test(self): self.plib.enable_region() regions = self.plib.run_ndctl_list('-R') self.plib.destroy_namespace(force=True) region = self.plib.run_ndctl_list_val(regions[0], 'dev') split = self.params.get('split_ns', default=False) if len(regions) == 1: if self.plib.is_region_legacy(region): self.cancel("Cannot create DM with single pmem device") if not split: self.cancel("Cannot run test without split option enabled") if split: if self.plib.is_region_legacy(region): self.cancel("Cannot split pmem device on legacy hardware") size_align = self.get_size_alignval() self.log.info("Creating namespace with existing regions") for reg_json in regions: region = self.plib.run_ndctl_list_val(reg_json, 'dev') slot_count = self.plib.get_slot_count(region) reg_size = self.plib.run_ndctl_list_val( self.plib.run_ndctl_list('-r %s' % region)[0], 'size') namespace_size = reg_size // slot_count # Now align the namespace size namespace_size = (namespace_size // size_align) * size_align if namespace_size <= size_align: self.log.warn("Skipping namespace size less than pagesize") continue for _ in range(0, slot_count): self.plib.create_namespace( region=region, size=namespace_size) else: self.log.info("Creating namespace with full size") for reg_json in regions: region = self.plib.run_ndctl_list_val(reg_json, 'dev') self.plib.create_namespace(region=region) devices = self.plib.run_ndctl_list('-N') blk_cmd = "" bdev = None blk_size1 = 0 for cnt, dev in enumerate(devices): bdev = self.plib.run_ndctl_list_val(dev, 'blockdev') bdev = "/dev/%s" % bdev blk_size2 = process.system_output( "blockdev --getsz %s" % bdev).decode() blk_cmd += ' %s %s linear %s 0 "\\\\n"' % ( blk_size1, blk_size2, bdev) blk_size1 += int(blk_size2) if cnt == len(devices) - 1: break dm_cmd = 'echo -e "%s" | dmsetup create linear-pmem' % blk_cmd if process.system(dm_cmd, shell=True, sudo=True, ignore_status=True): self.fail("Creating DM failed") self.log.info("Running FIO on device-mapper") dm_disk = "/dev/mapper/linear-pmem" self.part = partition.Partition(dm_disk) self.part.mkfs(fstype='xfs', args='-b size=%s -s size=512 -m reflink=0' % memory.get_page_size()) mnt_path = self.params.get('mnt_point', default='/pmem') if not os.path.exists(mnt_path): os.makedirs(mnt_path) self.part.mount(mountpoint=mnt_path, args='-o dax') self.log.info("Test will run on %s", mnt_path) fio_job = self.params.get('fio_job', default='ndctl-fio.job') size = disk.freespace(mnt_path) * 0.9 cmd = '%s --directory %s --filename mmap-pmem --size %s %s' % ( self.build_fio(), mnt_path, size, self.get_data(fio_job)) if process.system(cmd, ignore_status=True): self.fail("FIO mmap workload on fsdax failed")
def setUp(self): """ Build xfstest Source: git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git """ self.use_dd = False root_fs = process.system_output( "df -T / | awk 'END {print $2}'", shell=True).decode("utf-8") if root_fs in ['ext3', 'ext4']: self.use_dd = True self.dev_type = self.params.get('type', default='loop') sm = SoftwareManager() self.detected_distro = distro.detect() packages = ['e2fsprogs', 'automake', 'gcc', 'quota', 'attr', 'make', 'xfsprogs', 'gawk'] if self.detected_distro.name in ['Ubuntu', 'debian']: packages.extend( ['xfslibs-dev', 'uuid-dev', 'libuuid1', 'libattr1-dev', 'libacl1-dev', 'libgdbm-dev', 'uuid-runtime', 'libaio-dev', 'fio', 'dbench', 'gettext', 'libinih-dev', 'liburcu-dev', 'libblkid-dev', 'liblzo2-dev', 'zlib1g-dev', 'e2fslibs-dev', 'asciidoc', 'xmlto', 'libzstd-dev', 'libudev-dev']) if self.detected_distro.version in ['14']: packages.extend(['libtool']) elif self.detected_distro.version in ['18', '20']: packages.extend(['libtool-bin', 'libgdbm-compat-dev']) else: packages.extend(['libtool-bin']) elif self.detected_distro.name in ['centos', 'fedora', 'rhel', 'SuSE']: if self.dev_type == 'nvdimm': packages.extend(['ndctl', 'parted']) if self.detected_distro.name == 'rhel': packages.extend(['daxctl']) packages.extend(['acl', 'bc', 'dump', 'indent', 'libtool', 'lvm2', 'xfsdump', 'psmisc', 'sed', 'libacl-devel', 'libattr-devel', 'libaio-devel', 'libuuid-devel', 'openssl-devel', 'xfsprogs-devel', 'gettext', 'libblkid-devel', 'lzo-devel', 'zlib-devel', 'e2fsprogs-devel', 'asciidoc', 'xmlto', 'libzstd-devel', 'systemd-devel', 'meson', 'gcc-c++']) if self.detected_distro.name == 'SuSE': packages.extend(['libbtrfs-devel', 'libcap-progs', 'liburcu-devel', 'libinih-devel']) else: packages.extend(['btrfs-progs-devel', 'userspace-rcu-devel']) packages_remove = ['indent', 'btrfs-progs-devel'] if self.detected_distro.name == 'rhel' and\ self.detected_distro.version.startswith('8'): packages = list(set(packages)-set(packages_remove)) elif self.detected_distro.name == 'rhel' and\ self.detected_distro.version.startswith('9'): packages_remove.extend(['dump']) packages = list(set(packages)-set(packages_remove)) if self.detected_distro.name in ['centos', 'fedora']: packages.extend(['fio', 'dbench']) else: self.cancel("test not supported in %s" % self.detected_distro.name) for package in packages: if not sm.check_installed(package) and not sm.install(package): self.cancel("Fail to install %s required for this test." % package) self.skip_dangerous = self.params.get('skip_dangerous', default=True) self.test_range = self.params.get('test_range', default=None) self.scratch_mnt = self.params.get( 'scratch_mnt', default='/mnt/scratch') self.test_mnt = self.params.get('test_mnt', default='/mnt/test') self.disk_mnt = self.params.get('disk_mnt', default='/mnt/loop_device') self.fs_to_test = self.params.get('fs', default='ext4') self.run_type = self.params.get('run_type', default='distro') if self.run_type == 'upstream': prefix = "/usr/local" bin_prefix = "/usr/local/bin" if self.detected_distro.name == 'SuSE': # SuSE has /sbin at a higher priority than /usr/local/bin # in $PATH, so install all the binaries in /sbin to make # sure they are picked up correctly by xfstests. # # We still install in /usr/local but binaries are kept in # /sbin bin_prefix = "/sbin" if self.fs_to_test == "ext4": # Build e2fs progs e2fsprogs_dir = os.path.join(self.teststmpdir, 'e2fsprogs') if not os.path.exists(e2fsprogs_dir): os.makedirs(e2fsprogs_dir) e2fsprogs_url = self.params.get('e2fsprogs_url') git.get_repo(e2fsprogs_url, destination_dir=e2fsprogs_dir) e2fsprogs_build_dir = os.path.join(e2fsprogs_dir, 'build') if not os.path.exists(e2fsprogs_build_dir): os.makedirs(e2fsprogs_build_dir) os.chdir(e2fsprogs_build_dir) process.run("../configure --prefix=%s --bindir=%s --sbindir=%s" % (prefix, bin_prefix, bin_prefix), verbose=True) build.make(e2fsprogs_build_dir) build.make(e2fsprogs_build_dir, extra_args='install') if self.fs_to_test == "xfs": if self.detected_distro.name in ['centos', 'fedora', 'rhel']: libini_path = process.run("ldconfig -p | grep libini", verbose=True, ignore_status=True) if not libini_path: # Build libini.h as it is needed for xfsprogs libini_dir = os.path.join(self.teststmpdir, 'libini') if not os.path.exists(libini_dir): os.makedirs(libini_dir) git.get_repo('https://github.com/benhoyt/inih', destination_dir=libini_dir) os.chdir(libini_dir) process.run("meson build", verbose=True) libini_build_dir = os.path.join(libini_dir, 'build') if os.path.exists(libini_build_dir): os.chdir(libini_build_dir) process.run("meson install", verbose=True) else: self.fail('Something went wrong while building \ libini. Please check the logs.') # Build xfs progs xfsprogs_dir = os.path.join(self.teststmpdir, 'xfsprogs') if not os.path.exists(xfsprogs_dir): os.makedirs(xfsprogs_dir) xfsprogs_url = self.params.get('xfsprogs_url') git.get_repo(xfsprogs_url, destination_dir=xfsprogs_dir) os.chdir(xfsprogs_dir) build.make(xfsprogs_dir) process.run("./configure --prefix=%s --bindir=%s --sbindir=%s" % (prefix, bin_prefix, bin_prefix), verbose=True) build.make(xfsprogs_dir, extra_args='install') if self.fs_to_test == "btrfs": # Build btrfs progs btrfsprogs_dir = os.path.join(self.teststmpdir, 'btrfsprogs') if not os.path.exists(btrfsprogs_dir): os.makedirs(btrfsprogs_dir) btrfsprogs_url = self.params.get('btrfsprogs_url') git.get_repo(btrfsprogs_url, destination_dir=btrfsprogs_dir) os.chdir(btrfsprogs_dir) process.run("./autogen.sh", verbose=True) process.run("./configure --prefix=%s --bindir=%s --sbindir=%s --disable-documentation" % (prefix, bin_prefix, bin_prefix), verbose=True) build.make(btrfsprogs_dir) build.make(btrfsprogs_dir, extra_args='install') # Check versions of fsprogs fsprogs_ver = process.system_output("mkfs.%s -V" % self.fs_to_test, ignore_status=True, shell=True).decode("utf-8") self.log.info(fsprogs_ver) if process.system('which mkfs.%s' % self.fs_to_test, ignore_status=True): self.cancel('Unknown filesystem %s' % self.fs_to_test) mount = True self.devices = [] self.log_devices = [] shutil.copyfile(self.get_data('local.config'), os.path.join(self.teststmpdir, 'local.config')) shutil.copyfile(self.get_data('group'), os.path.join(self.teststmpdir, 'group')) self.log_test = self.params.get('log_test', default='') self.log_scratch = self.params.get('log_scratch', default='') if self.dev_type == 'loop': base_disk = self.params.get('disk', default=None) loop_size = self.params.get('loop_size', default='7GiB') if not base_disk: # Using root for file creation by default check = (int(loop_size.split('GiB')[0]) * 2) + 1 if disk.freespace('/') / 1073741824 > check: self.disk_mnt = '' mount = False else: self.cancel('Need %s GB to create loop devices' % check) self._create_loop_device(base_disk, loop_size, mount) elif self.dev_type == 'nvdimm': self.setup_nvdimm() else: self.test_dev = self.params.get('disk_test', default=None) self.scratch_dev = self.params.get('disk_scratch', default=None) self.devices.extend([self.test_dev, self.scratch_dev]) # mkfs for devices if self.devices: cfg_file = os.path.join(self.teststmpdir, 'local.config') self.mkfs_opt = self.params.get('mkfs_opt', default='') self.mount_opt = self.params.get('mount_opt', default='') with open(cfg_file, "r") as sources: lines = sources.readlines() with open(cfg_file, "w") as sources: for line in lines: if line.startswith('export TEST_DEV'): sources.write( re.sub(r'export TEST_DEV=.*', 'export TEST_DEV=%s' % self.devices[0], line)) elif line.startswith('export TEST_DIR'): sources.write( re.sub(r'export TEST_DIR=.*', 'export TEST_DIR=%s' % self.test_mnt, line)) elif line.startswith('export SCRATCH_DEV'): sources.write(re.sub( r'export SCRATCH_DEV=.*', 'export SCRATCH_DEV=%s' % self.devices[1], line)) elif line.startswith('export SCRATCH_MNT'): sources.write( re.sub( r'export SCRATCH_MNT=.*', 'export SCRATCH_MNT=%s' % self.scratch_mnt, line)) break with open(cfg_file, "a") as sources: if self.log_test: sources.write('export USE_EXTERNAL=yes\n') sources.write('export TEST_LOGDEV="%s"\n' % self.log_test) self.log_devices.append(self.log_test) if self.log_scratch: sources.write('export SCRATCH_LOGDEV="%s"\n' % self.log_scratch) self.log_devices.append(self.log_scratch) if self.mkfs_opt: sources.write('MKFS_OPTIONS="%s"\n' % self.mkfs_opt) if self.mount_opt: sources.write('MOUNT_OPTIONS="%s"\n' % self.mount_opt) self.logdev_opt = self.params.get('logdev_opt', default='') for dev in self.log_devices: dev_obj = partition.Partition(dev) dev_obj.mkfs(fstype=self.fs_to_test, args=self.mkfs_opt) for ite, dev in enumerate(self.devices): dev_obj = partition.Partition(dev) if self.logdev_opt: dev_obj.mkfs(fstype=self.fs_to_test, args='%s %s=%s' % ( self.mkfs_opt, self.logdev_opt, self.log_devices[ite])) else: dev_obj.mkfs(fstype=self.fs_to_test, args=self.mkfs_opt) git.get_repo('git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git', destination_dir=self.teststmpdir) build.make(self.teststmpdir) self.available_tests = self._get_available_tests() self.test_list = self._create_test_list(self.test_range) self.log.info("Tests available in srcdir: %s", ", ".join(self.available_tests)) if not self.test_range: self.exclude = self.params.get('exclude', default=None) self.gen_exclude = self.params.get('gen_exclude', default=None) self.share_exclude = self.params.get('share_exclude', default=None) if self.exclude or self.gen_exclude or self.share_exclude: self.exclude_file = os.path.join(self.teststmpdir, 'exclude') if self.exclude: self._create_test_list(self.exclude, self.fs_to_test, dangerous=False) if self.gen_exclude: self._create_test_list(self.gen_exclude, "generic", dangerous=False) if self.share_exclude: self._create_test_list(self.share_exclude, "shared", dangerous=False) if self.detected_distro.name is not 'SuSE': if process.system('useradd 123456-fsgqa', sudo=True, ignore_status=True): self.log.warn('useradd 123456-fsgqa failed') if process.system('useradd fsgqa', sudo=True, ignore_status=True): self.log.warn('useradd fsgqa failed') else: if process.system('useradd -m -U fsgqa', sudo=True, ignore_status=True): self.log.warn('useradd fsgqa failed') if process.system('groupadd sys', sudo=True, ignore_status=True): self.log.warn('groupadd sys failed') if not os.path.exists(self.scratch_mnt): os.makedirs(self.scratch_mnt) if not os.path.exists(self.test_mnt): os.makedirs(self.test_mnt)
def setUp(self): """ Build xfstest Source: git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git """ self.use_dd = False root_fs = process.system_output("df -T / | awk 'END {print $2}'", shell=True).decode("utf-8") if root_fs in ['ext3', 'ext4']: self.use_dd = True self.dev_type = self.params.get('type', default='loop') sm = SoftwareManager() self.detected_distro = distro.detect() packages = [ 'e2fsprogs', 'automake', 'gcc', 'quota', 'attr', 'make', 'xfsprogs', 'gawk' ] if self.detected_distro.name in ['Ubuntu', 'debian']: packages.extend([ 'xfslibs-dev', 'uuid-dev', 'libuuid1', 'libattr1-dev', 'libacl1-dev', 'libgdbm-dev', 'uuid-runtime', 'libaio-dev', 'fio', 'dbench', 'libbtrfs-dev' ]) if self.detected_distro.version in ['14']: packages.extend(['libtool']) elif self.detected_distro.version in ['18', '20']: packages.extend(['libtool-bin', 'libgdbm-compat-dev']) else: packages.extend(['libtool-bin']) # FIXME: "redhat" as the distro name for RHEL is deprecated # on Avocado versions >= 50.0. This is a temporary compatibility # enabler for older runners, but should be removed soon elif self.detected_distro.name in ['centos', 'fedora', 'rhel', 'SuSE']: if self.dev_type == 'nvdimm': packages.extend(['ndctl', 'parted']) if self.detected_distro.name == 'rhel': packages.extend(['daxctl']) packages.extend([ 'acl', 'bc', 'dump', 'indent', 'libtool', 'lvm2', 'xfsdump', 'psmisc', 'sed', 'libacl-devel', 'libattr-devel', 'libaio-devel', 'libuuid-devel', 'openssl-devel', 'xfsprogs-devel' ]) if self.detected_distro.name == 'SuSE': packages.extend(['libbtrfs-devel', 'libcap-progs']) else: packages.extend(['btrfs-progs-devel']) if self.detected_distro.name == 'rhel' and\ self.detected_distro.version.startswith('8'): packages.remove('indent') packages.remove('btrfs-progs-devel') if self.detected_distro.name in ['centos', 'fedora']: packages.extend(['fio', 'dbench']) else: self.cancel("test not supported in %s" % self.detected_distro.name) for package in packages: if not sm.check_installed(package) and not sm.install(package): self.cancel("Fail to install %s required for this test." % package) self.skip_dangerous = self.params.get('skip_dangerous', default=True) self.test_range = self.params.get('test_range', default=None) self.scratch_mnt = self.params.get('scratch_mnt', default='/mnt/scratch') self.test_mnt = self.params.get('test_mnt', default='/mnt/test') self.disk_mnt = self.params.get('disk_mnt', default='/mnt/loop_device') self.fs_to_test = self.params.get('fs', default='ext4') if process.system('which mkfs.%s' % self.fs_to_test, ignore_status=True): self.cancel('Unknown filesystem %s' % self.fs_to_test) mount = True self.devices = [] self.log_devices = [] shutil.copyfile(self.get_data('local.config'), os.path.join(self.teststmpdir, 'local.config')) shutil.copyfile(self.get_data('group'), os.path.join(self.teststmpdir, 'group')) if self.dev_type == 'loop': base_disk = self.params.get('disk', default=None) loop_size = self.params.get('loop_size', default='7GiB') if not base_disk: # Using root for file creation by default check = (int(loop_size.split('GiB')[0]) * 2) + 1 if disk.freespace('/') / 1073741824 > check: self.disk_mnt = '' mount = False else: self.cancel('Need %s GB to create loop devices' % check) self._create_loop_device(base_disk, loop_size, mount) elif self.dev_type == 'nvdimm': self.test_dev = self.params.get('disk_test', default=None) self.scratch_dev = self.params.get('disk_scratch', default=None) self.log_test = self.params.get('log_test', default='') self.log_scratch = self.params.get('log_scratch', default='') logflag = self.params.get('logdev', default=False) self.plib = pmem.PMem() regions = sorted(self.plib.run_ndctl_list('-R'), key=lambda i: i['size'], reverse=True) if not (self.test_dev and self.scratch_dev): if not regions: self.plib.enable_region() regions = sorted(self.plib.run_ndctl_list('-R'), key=lambda i: i['size'], reverse=True) region = self.plib.run_ndctl_list_val(regions[0], 'dev') if not self.plib.is_region_legacy(region): self.plib.destroy_namespace(region=region, force=True) self.plib.create_namespace(region=region, size='21G', sector_size='512') pmem_dev = self.plib.run_ndctl_list_val( self.plib.run_ndctl_list('-N -r %s' % region)[0], 'blockdev') if logflag: if not (self.log_test and self.log_scratch): self.plib.create_namespace(region=region, size='3G', mode='sector', sector_size='512') log_dev = self.plib.run_ndctl_list_val( self.plib.run_ndctl_list('-N -r %s -m sector' % region)[0], 'blockdev') else: self.plib.enable_region() if not len(regions) > 1 and logflag: self.cancel("Cannot use logdev with one region") else: reg_2 = self.plib.run_ndctl_list_val(regions[1], 'dev') self.plib.create_namespace(region=reg_2, size='3G', mode='sector', sector_size='512') log_dev = self.plib.run_ndctl_list_val( self.plib.run_ndctl_list(' -N -r %s -m sector' % reg_2)[0], 'blockdev') pmem_dev = self.plib.run_ndctl_list_val( self.plib.run_ndctl_list('-N -r %s' % region)[0], 'blockdev') if process.system('parted -s -a optimal /dev/%s mklabel gpt --' ' mkpart primary xfs 1MiB 10GiB mkpart ' 'primary xfs 10GiB 20GiB' % pmem_dev, shell=True): self.cancel("Failed to setup PMEM partitions") self.test_dev = "/dev/%sp1" % pmem_dev self.scratch_dev = "/dev/%sp2" % pmem_dev self.devices.extend([self.test_dev, self.scratch_dev]) if logflag and not (self.log_test and self.log_scratch): if process.system('parted -s -a optimal /dev/%s mklabel gpt --' ' mkpart primary xfs 1MiB 1GiB mkpart ' 'primary xfs 1GiB 2GiB' % log_dev, shell=True): self.cancel("Failed to setup logdev partitions") self.log_test = "/dev/%s1" % log_dev self.log_scratch = "/dev/%s2" % log_dev else: self.test_dev = self.params.get('disk_test', default=None) self.scratch_dev = self.params.get('disk_scratch', default=None) self.devices.extend([self.test_dev, self.scratch_dev]) # mkfs for devices if self.devices: cfg_file = os.path.join(self.teststmpdir, 'local.config') self.mkfs_opt = self.params.get('mkfs_opt', default='') self.mount_opt = self.params.get('mount_opt', default='') with open(cfg_file, "r") as sources: lines = sources.readlines() with open(cfg_file, "w") as sources: for line in lines: if line.startswith('export TEST_DEV'): sources.write( re.sub(r'export TEST_DEV=.*', 'export TEST_DEV=%s' % self.devices[0], line)) elif line.startswith('export TEST_DIR'): sources.write( re.sub(r'export TEST_DIR=.*', 'export TEST_DIR=%s' % self.test_mnt, line)) elif line.startswith('export SCRATCH_DEV'): sources.write( re.sub(r'export SCRATCH_DEV=.*', 'export SCRATCH_DEV=%s' % self.devices[1], line)) elif line.startswith('export SCRATCH_MNT'): sources.write( re.sub(r'export SCRATCH_MNT=.*', 'export SCRATCH_MNT=%s' % self.scratch_mnt, line)) break with open(cfg_file, "a") as sources: if self.log_test: sources.write('export USE_EXTERNAL=yes\n') sources.write('export TEST_LOGDEV="%s"\n' % self.log_test) self.log_devices.append(self.log_test) if self.log_scratch: sources.write('export SCRATCH_LOGDEV="%s"\n' % self.log_scratch) self.log_devices.append(self.log_scratch) if self.mkfs_opt: sources.write('MKFS_OPTIONS="%s"\n' % self.mkfs_opt) if self.mount_opt: sources.write('MOUNT_OPTIONS="%s"\n' % self.mount_opt) self.logdev_opt = self.params.get('logdev_opt', default='') for dev in self.log_devices: dev_obj = partition.Partition(dev) dev_obj.mkfs(fstype=self.fs_to_test, args=self.mkfs_opt) for ite, dev in enumerate(self.devices): dev_obj = partition.Partition(dev) if self.logdev_opt: dev_obj.mkfs(fstype=self.fs_to_test, args='%s %s=%s' % (self.mkfs_opt, self.logdev_opt, self.log_devices[ite])) else: dev_obj.mkfs(fstype=self.fs_to_test, args=self.mkfs_opt) git.get_repo('git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git', destination_dir=self.teststmpdir) build.make(self.teststmpdir) self.available_tests = self._get_available_tests() self.test_list = self._create_test_list(self.test_range) self.log.info("Tests available in srcdir: %s", ", ".join(self.available_tests)) if not self.test_range: self.exclude = self.params.get('exclude', default=None) self.gen_exclude = self.params.get('gen_exclude', default=None) self.share_exclude = self.params.get('share_exclude', default=None) if self.exclude or self.gen_exclude or self.share_exclude: self.exclude_file = os.path.join(self.teststmpdir, 'exclude') if self.exclude: self._create_test_list(self.exclude, self.fs_to_test, dangerous=False) if self.gen_exclude: self._create_test_list(self.gen_exclude, "generic", dangerous=False) if self.share_exclude: self._create_test_list(self.share_exclude, "shared", dangerous=False) if self.detected_distro.name is not 'SuSE': if process.system('useradd 123456-fsgqa', sudo=True, ignore_status=True): self.log.warn('useradd 123456-fsgqa failed') if process.system('useradd fsgqa', sudo=True, ignore_status=True): self.log.warn('useradd fsgqa failed') else: if process.system('useradd -m -U fsgqa', sudo=True, ignore_status=True): self.log.warn('useradd fsgqa failed') if process.system('groupadd sys', sudo=True, ignore_status=True): self.log.warn('groupadd sys failed') if not os.path.exists(self.scratch_mnt): os.makedirs(self.scratch_mnt) if not os.path.exists(self.test_mnt): os.makedirs(self.test_mnt)
def setUp(self): """ Build xfstest Source: git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git """ self.use_dd = False root_fs = process.system_output("df -T / | awk 'END {print $2}'", shell=True).decode("utf-8") if root_fs in ['ext3', 'ext4']: self.use_dd = True self.dev_type = self.params.get('type', default='loop') sm = SoftwareManager() self.detected_distro = distro.detect() packages = [ 'e2fsprogs', 'automake', 'gcc', 'quota', 'attr', 'make', 'xfsprogs', 'gawk' ] if self.detected_distro.name in ['Ubuntu', 'debian']: packages.extend([ 'xfslibs-dev', 'uuid-dev', 'libuuid1', 'libattr1-dev', 'libacl1-dev', 'libgdbm-dev', 'uuid-runtime', 'libaio-dev', 'fio', 'dbench' ]) if self.detected_distro.version in ['14']: packages.extend(['libtool']) elif self.detected_distro.version in ['18', '20']: packages.extend(['libtool-bin', 'libgdbm-compat-dev']) else: packages.extend(['libtool-bin']) elif self.detected_distro.name in ['centos', 'fedora', 'rhel', 'SuSE']: if self.dev_type == 'nvdimm': packages.extend(['ndctl', 'parted']) if self.detected_distro.name == 'rhel': packages.extend(['daxctl']) packages.extend([ 'acl', 'bc', 'dump', 'indent', 'libtool', 'lvm2', 'xfsdump', 'psmisc', 'sed', 'libacl-devel', 'libattr-devel', 'libaio-devel', 'libuuid-devel', 'openssl-devel', 'xfsprogs-devel' ]) if self.detected_distro.name == 'SuSE': packages.extend(['libbtrfs-devel', 'libcap-progs']) else: packages.extend(['btrfs-progs-devel']) packages_remove = ['indent', 'btrfs-progs-devel'] if self.detected_distro.name == 'rhel' and\ self.detected_distro.version.startswith('8'): packages = list(set(packages) - set(packages_remove)) elif self.detected_distro.name == 'rhel' and\ self.detected_distro.version.startswith('9'): packages_remove.extend(['dump']) packages = list(set(packages) - set(packages_remove)) if self.detected_distro.name in ['centos', 'fedora']: packages.extend(['fio', 'dbench']) else: self.cancel("test not supported in %s" % self.detected_distro.name) for package in packages: if not sm.check_installed(package) and not sm.install(package): self.cancel("Fail to install %s required for this test." % package) self.skip_dangerous = self.params.get('skip_dangerous', default=True) self.test_range = self.params.get('test_range', default=None) self.scratch_mnt = self.params.get('scratch_mnt', default='/mnt/scratch') self.test_mnt = self.params.get('test_mnt', default='/mnt/test') self.disk_mnt = self.params.get('disk_mnt', default='/mnt/loop_device') self.fs_to_test = self.params.get('fs', default='ext4') if process.system('which mkfs.%s' % self.fs_to_test, ignore_status=True): self.cancel('Unknown filesystem %s' % self.fs_to_test) mount = True self.devices = [] self.log_devices = [] shutil.copyfile(self.get_data('local.config'), os.path.join(self.teststmpdir, 'local.config')) shutil.copyfile(self.get_data('group'), os.path.join(self.teststmpdir, 'group')) self.log_test = self.params.get('log_test', default='') self.log_scratch = self.params.get('log_scratch', default='') if self.dev_type == 'loop': base_disk = self.params.get('disk', default=None) loop_size = self.params.get('loop_size', default='7GiB') if not base_disk: # Using root for file creation by default check = (int(loop_size.split('GiB')[0]) * 2) + 1 if disk.freespace('/') / 1073741824 > check: self.disk_mnt = '' mount = False else: self.cancel('Need %s GB to create loop devices' % check) self._create_loop_device(base_disk, loop_size, mount) elif self.dev_type == 'nvdimm': self.setup_nvdimm() else: self.test_dev = self.params.get('disk_test', default=None) self.scratch_dev = self.params.get('disk_scratch', default=None) self.devices.extend([self.test_dev, self.scratch_dev]) # mkfs for devices if self.devices: cfg_file = os.path.join(self.teststmpdir, 'local.config') self.mkfs_opt = self.params.get('mkfs_opt', default='') self.mount_opt = self.params.get('mount_opt', default='') with open(cfg_file, "r") as sources: lines = sources.readlines() with open(cfg_file, "w") as sources: for line in lines: if line.startswith('export TEST_DEV'): sources.write( re.sub(r'export TEST_DEV=.*', 'export TEST_DEV=%s' % self.devices[0], line)) elif line.startswith('export TEST_DIR'): sources.write( re.sub(r'export TEST_DIR=.*', 'export TEST_DIR=%s' % self.test_mnt, line)) elif line.startswith('export SCRATCH_DEV'): sources.write( re.sub(r'export SCRATCH_DEV=.*', 'export SCRATCH_DEV=%s' % self.devices[1], line)) elif line.startswith('export SCRATCH_MNT'): sources.write( re.sub(r'export SCRATCH_MNT=.*', 'export SCRATCH_MNT=%s' % self.scratch_mnt, line)) break with open(cfg_file, "a") as sources: if self.log_test: sources.write('export USE_EXTERNAL=yes\n') sources.write('export TEST_LOGDEV="%s"\n' % self.log_test) self.log_devices.append(self.log_test) if self.log_scratch: sources.write('export SCRATCH_LOGDEV="%s"\n' % self.log_scratch) self.log_devices.append(self.log_scratch) if self.mkfs_opt: sources.write('MKFS_OPTIONS="%s"\n' % self.mkfs_opt) if self.mount_opt: sources.write('MOUNT_OPTIONS="%s"\n' % self.mount_opt) self.logdev_opt = self.params.get('logdev_opt', default='') for dev in self.log_devices: dev_obj = partition.Partition(dev) dev_obj.mkfs(fstype=self.fs_to_test, args=self.mkfs_opt) for ite, dev in enumerate(self.devices): dev_obj = partition.Partition(dev) if self.logdev_opt: dev_obj.mkfs(fstype=self.fs_to_test, args='%s %s=%s' % (self.mkfs_opt, self.logdev_opt, self.log_devices[ite])) else: dev_obj.mkfs(fstype=self.fs_to_test, args=self.mkfs_opt) git.get_repo('git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git', destination_dir=self.teststmpdir) build.make(self.teststmpdir) self.available_tests = self._get_available_tests() self.test_list = self._create_test_list(self.test_range) self.log.info("Tests available in srcdir: %s", ", ".join(self.available_tests)) if not self.test_range: self.exclude = self.params.get('exclude', default=None) self.gen_exclude = self.params.get('gen_exclude', default=None) self.share_exclude = self.params.get('share_exclude', default=None) if self.exclude or self.gen_exclude or self.share_exclude: self.exclude_file = os.path.join(self.teststmpdir, 'exclude') if self.exclude: self._create_test_list(self.exclude, self.fs_to_test, dangerous=False) if self.gen_exclude: self._create_test_list(self.gen_exclude, "generic", dangerous=False) if self.share_exclude: self._create_test_list(self.share_exclude, "shared", dangerous=False) if self.detected_distro.name is not 'SuSE': if process.system('useradd 123456-fsgqa', sudo=True, ignore_status=True): self.log.warn('useradd 123456-fsgqa failed') if process.system('useradd fsgqa', sudo=True, ignore_status=True): self.log.warn('useradd fsgqa failed') else: if process.system('useradd -m -U fsgqa', sudo=True, ignore_status=True): self.log.warn('useradd fsgqa failed') if process.system('groupadd sys', sudo=True, ignore_status=True): self.log.warn('groupadd sys failed') if not os.path.exists(self.scratch_mnt): os.makedirs(self.scratch_mnt) if not os.path.exists(self.test_mnt): os.makedirs(self.test_mnt)
def setUp(self): """ Build xfstest Source: git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git """ self.use_dd = False root_fs = process.system_output("df -T / | awk 'END {print $2}'", shell=True) if root_fs in ['ext3', 'ext4']: self.use_dd = True sm = SoftwareManager() self.detected_distro = distro.detect() packages = [ 'e2fsprogs', 'automake', 'gcc', 'quota', 'attr', 'make', 'xfsprogs', 'gawk' ] if 'Ubuntu' in self.detected_distro.name: packages.extend([ 'xfslibs-dev', 'uuid-dev', 'libuuid1', 'libattr1-dev', 'libacl1-dev', 'libgdbm-dev', 'uuid-runtime', 'libaio-dev', 'fio', 'dbench', 'btrfs-tools' ]) if '14' in self.detected_distro.version: packages.extend(['libtool']) elif '18' in self.detected_distro.version: packages.extend(['libtool-bin', 'libgdbm-compat-dev']) else: packages.extend(['libtool-bin']) # FIXME: "redhat" as the distro name for RHEL is deprecated # on Avocado versions >= 50.0. This is a temporary compatibility # enabler for older runners, but should be removed soon elif self.detected_distro.name in [ 'centos', 'fedora', 'rhel', 'redhat', 'SuSE' ]: packages.extend([ 'acl', 'bc', 'dump', 'indent', 'libtool', 'lvm2', 'xfsdump', 'psmisc', 'sed', 'libacl-devel', 'libattr-devel', 'libaio-devel', 'libuuid-devel', 'openssl-devel', 'xfsprogs-devel' ]) if self.detected_distro.name == 'SuSE': packages.extend(['libbtrfs-devel', 'libcap-progs']) else: packages.extend(['btrfs-progs-devel']) if self.detected_distro.name in ['centos', 'fedora']: packages.extend(['fio', 'dbench']) else: self.cancel("test not supported in %s" % self.detected_distro.name) for package in packages: if not sm.check_installed(package) and not sm.install(package): self.cancel("Fail to install %s required for this test." % package) self.skip_dangerous = self.params.get('skip_dangerous', default=True) self.test_range = self.params.get('test_range', default=None) self.scratch_mnt = self.params.get('scratch_mnt', default='/mnt/scratch') self.test_mnt = self.params.get('test_mnt', default='/mnt/test') self.disk_mnt = self.params.get('disk_mnt', default='/mnt/loop_device') self.dev_type = self.params.get('type', default='loop') self.fs_to_test = self.params.get('fs', default='ext4') if process.system('which mkfs.%s' % self.fs_to_test, ignore_status=True): self.cancel('Unknown filesystem %s' % self.fs_to_test) mount = True self.devices = [] shutil.copyfile(self.get_data('local.config'), os.path.join(self.teststmpdir, 'local.config')) shutil.copyfile(self.get_data('group'), os.path.join(self.teststmpdir, 'group')) if self.dev_type == 'loop': base_disk = self.params.get('disk', default=None) loop_size = self.params.get('loop_size', default='7GiB') if not base_disk: # Using root for file creation by default check = (int(loop_size.split('GiB')[0]) * 2) + 1 if disk.freespace('/') / 1073741824 > check: self.disk_mnt = '' mount = False else: self.cancel('Need %s GB to create loop devices' % check) self._create_loop_device(base_disk, loop_size, mount) else: self.test_dev = self.params.get('disk_test', default=None) self.scratch_dev = self.params.get('disk_scratch', default=None) self.devices.extend([self.test_dev, self.scratch_dev]) # mkfs for devices if self.devices: cfg_file = os.path.join(self.teststmpdir, 'local.config') with open(cfg_file, "r") as sources: lines = sources.readlines() with open(cfg_file, "w") as sources: for line in lines: if line.startswith('export TEST_DEV'): sources.write( re.sub(r'export TEST_DEV=.*', 'export TEST_DEV=%s' % self.devices[0], line)) elif line.startswith('export TEST_DIR'): sources.write( re.sub(r'export TEST_DIR=.*', 'export TEST_DIR=%s' % self.test_mnt, line)) elif line.startswith('export SCRATCH_DEV'): sources.write( re.sub(r'export SCRATCH_DEV=.*', 'export SCRATCH_DEV=%s' % self.devices[1], line)) elif line.startswith('export SCRATCH_MNT'): sources.write( re.sub(r'export SCRATCH_MNT=.*', 'export SCRATCH_MNT=%s' % self.scratch_mnt, line)) break for dev in self.devices: dev_obj = partition.Partition(dev) dev_obj.mkfs(fstype=self.fs_to_test) git.get_repo('git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git', destination_dir=self.teststmpdir) build.make(self.teststmpdir) self.available_tests = self._get_available_tests() self.test_list = self._create_test_list(self.test_range) self.log.info("Tests available in srcdir: %s", ", ".join(self.available_tests)) if not self.test_range: self.exclude = self.params.get('exclude', default=None) self.gen_exclude = self.params.get('gen_exclude', default=None) self.share_exclude = self.params.get('share_exclude', default=None) if self.exclude or self.gen_exclude or self.share_exclude: self.exclude_file = os.path.join(self.teststmpdir, 'exclude') if self.exclude: self._create_test_list(self.exclude, self.fs_to_test, dangerous=False) if self.gen_exclude: self._create_test_list(self.gen_exclude, "generic", dangerous=False) if self.share_exclude: self._create_test_list(self.share_exclude, "shared", dangerous=False) if self.detected_distro.name is not 'SuSE': process.run('useradd 123456-fsgqa', sudo=True) process.run('useradd fsgqa', sudo=True) else: process.run('useradd -m -U fsgqa', sudo=True) process.run('groupadd sys', sudo=True) if not os.path.exists(self.scratch_mnt): os.makedirs(self.scratch_mnt) if not os.path.exists(self.test_mnt): os.makedirs(self.test_mnt)
def setup_nvdimm(self): self.logflag = self.params.get('logdev', default=False) self.plib = pmem.PMem() self.plib.enable_region() regions = sorted(self.plib.run_ndctl_list('-R'), key=lambda i: i['size'], reverse=True) if not regions: self.cancel("Nvdimm test with no region support") self.region = self.plib.run_ndctl_list_val(regions[0], 'dev') if self.plib.is_region_legacy(self.region): if not len(regions) > 1: self.cancel("Not supported with single legacy region") if self.logflag: self.log.info("Using loop devices as log devices") check = 2 mount = True if disk.freespace('/') / 1073741824 > check: self.disk_mnt = '' mount = False else: self.cancel('Need %s GB to create loop devices' % check) self._create_loop_device(None, '2038M', mount) self.log_test = self.devices.pop() self.log_scratch = self.devices.pop() namespaces = self.plib.run_ndctl_list('-N -r %s' % self.region) pmem_dev = self.plib.run_ndctl_list_val(namespaces[0], 'blockdev') self.test_dev = "/dev/%s" % pmem_dev region_2 = self.plib.run_ndctl_list_val(regions[1], 'dev') namespaces = self.plib.run_ndctl_list('-N -r %s' % region_2) pmem_dev = self.plib.run_ndctl_list_val(namespaces[0], 'blockdev') self.scratch_dev = "/dev/%s" % pmem_dev self.devices.extend([self.test_dev, self.scratch_dev]) else: if self.logflag: if not len(regions) > 1: self.log.info('Using 10% space of device for logdev') self.region_ldev = self.region region_size = self.plib.run_ndctl_list_val( self.plib.run_ndctl_list('-r %s' % self.region_ldev)[0], 'size') logdev_size = int(region_size * 0.10) dev_size = region_size - logdev_size size_align = self.get_size_alignval() dev_size = dev_size // 2 dev_size = (dev_size // size_align) * size_align logdev_size = logdev_size // 2 logdev_size = (logdev_size // size_align) * size_align else: dev_size = self.get_half_region_size(self.region) self.region_ldev = self.plib.run_ndctl_list_val( regions[1], 'dev') logdev_size = self.get_half_region_size( region=self.region_ldev) self.plib.destroy_namespace(region=self.region, force=True) self.plib.destroy_namespace(region=self.region_ldev, force=True) # XFS restrict max log size to 2136997888, which is 10M less # than 2GB, not 16M page-aligned, hence rounding-off to nearest # 16M align value 2130706432, which is 16M less than 2GiB logdev_size = min(logdev_size, 2130706432) # log device to be created in sector mode self.plib.create_namespace(region=self.region_ldev, mode='sector', sector_size='512', size=logdev_size) self.plib.create_namespace(region=self.region_ldev, mode='sector', sector_size='512', size=logdev_size) namespaces = self.plib.run_ndctl_list('-N -r %s -m sector' % self.region_ldev) log_dev = self.plib.run_ndctl_list_val(namespaces[0], 'blockdev') self.log_test = "/dev/%s" % log_dev log_dev = self.plib.run_ndctl_list_val(namespaces[1], 'blockdev') self.log_scratch = "/dev/%s" % log_dev else: self.plib.destroy_namespace(region=self.region, force=True) dev_size = self.get_half_region_size(self.region) self.log_test = None self.log_scratch = None self.plib.create_namespace(region=self.region, size=dev_size) self.plib.create_namespace(region=self.region, size=dev_size) namespaces = self.plib.run_ndctl_list('-N -r %s -m fsdax' % self.region) pmem_dev = self.plib.run_ndctl_list_val(namespaces[0], 'blockdev') self.test_dev = "/dev/%s" % pmem_dev pmem_dev = self.plib.run_ndctl_list_val(namespaces[1], 'blockdev') self.scratch_dev = "/dev/%s" % pmem_dev self.devices.extend([self.test_dev, self.scratch_dev])
def setUp(self): """ Build xfstest Source: git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git """ if process.system_output("df -T / | awk 'END {print $2}'", shell=True) == 'ext3': self.skip('Test does not support ext3 root file system') sm = SoftwareManager() self.detected_distro = distro.detect() packages = [ 'e2fsprogs', 'automake', 'gcc', 'quota', 'attr', 'make', 'xfsprogs', 'gawk' ] if 'Ubuntu' in self.detected_distro.name: packages.extend([ 'xfslibs-dev', 'uuid-dev', 'libtool-bin', 'libuuid1', 'libattr1-dev', 'libacl1-dev', 'libgdbm-dev', 'uuid-runtime', 'libaio-dev', 'fio', 'dbench' ]) # FIXME: "redhat" as the distro name for RHEL is deprecated # on Avocado versions >= 50.0. This is a temporary compatibility # enabler for older runners, but should be removed soon elif self.detected_distro.name in [ 'centos', 'fedora', 'rhel', 'redhat', 'SuSE' ]: packages.extend([ 'acl', 'bc', 'dump', 'indent', 'libtool', 'lvm2', 'xfsdump', 'psmisc', 'sed', 'libacl-devel', 'libattr-devel', 'libaio-devel', 'libuuid-devel', 'openssl-devel', 'xfsprogs-devel' ]) if self.detected_distro.name == 'SuSE': packages.extend(['libbtrfs-devel']) else: packages.extend(['btrfs-progs-devel']) if self.detected_distro.name in ['centos', 'fedora']: packages.extend(['fio', 'dbench']) else: self.skip("test not supported in %s" % self.detected_distro.name) for package in packages: if not sm.check_installed(package) and not sm.install(package): self.skip("Fail to install %s required for this test." % package) self.skip_dangerous = self.params.get('skip_dangerous', default=True) self.test_range = self.params.get('test_range', default=None) self.scratch_mnt = self.params.get('scratch_mnt', default='/mnt/scratch') self.test_mnt = self.params.get('test_mnt', default='/mnt/test') self.disk_mnt = self.params.get('disk_mnt', default='/mnt/loop_device') self.dev_type = self.params.get('type', default='loop') self.fs_to_test = self.params.get('fs', default='ext4') if process.system('which mkfs.%s' % self.fs_to_test, ignore_status=True): self.skip('Unknown filesystem %s' % self.fs_to_test) mount = True self.devices = [] shutil.copyfile(os.path.join(self.datadir, 'local.config'), os.path.join(self.srcdir, 'local.config')) shutil.copyfile(os.path.join(self.datadir, 'group'), os.path.join(self.srcdir, 'group')) if self.dev_type == 'loop': base_disk = self.params.get('disk', default=None) loop_size = self.params.get('loop_size', default='9GiB') if not base_disk: # Using root for file creation by default if disk.freespace('/') / 1073741824 > 15: self.disk_mnt = '' mount = False else: self.skip('Need 15 GB to create loop devices') self._create_loop_device(base_disk, loop_size, mount) else: self.test_dev = self.params.get('disk_test', default=None) self.scratch_dev = self.params.get('disk_scratch', default=None) self.devices.extend([self.test_dev, self.scratch_dev]) # mkfs for devices if self.devices: line = ('export TEST_DEV=%s' % self.devices[0]).replace('/', '\/') process.system('sed -i "s/export TEST_DEV=.*/%s/g" %s' % (line, os.path.join(self.srcdir, 'local.config')), shell=True) line = ('export SCRATCH_DEV=%s' % self.devices[1]).replace( '/', '\/') process.system('sed -i "s/export SCRATCH_DEV=.*/%s/g" %s' % (line, os.path.join(self.srcdir, 'local.config')), shell=True) for dev in self.devices: dev_obj = partition.Partition(dev) dev_obj.mkfs(fstype=self.fs_to_test) git.get_repo('git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git', destination_dir=self.srcdir) build.make(self.srcdir) self.available_tests = self._get_available_tests() self.test_list = self._create_test_list() self.log.info("Tests available in srcdir: %s", ", ".join(self.available_tests)) process.run('useradd fsgqa', sudo=True) if self.detected_distro.name is not 'SuSE': process.run('useradd 123456-fsgqa', sudo=True) if not os.path.exists(self.scratch_mnt): os.makedirs(self.scratch_mnt) if not os.path.exists(self.test_mnt): os.makedirs(self.test_mnt)