def test_badly_behaved(self):
"""
Make sure avocado can cleanly get out of a loop of badly behaved tests.
"""
bad_test_basename = ('wontquit-%s' %
data_factory.generate_random_string(5))
bad_test = script.TemporaryScript(bad_test_basename, BAD_TEST,
'avocado_interrupt_test',
mode=0755)
bad_test.save()
os.chdir(basedir)
cmd_line = ('./scripts/avocado run --sysinfo=off --job-results-dir %s '
'%s %s %s' % (self.tmpdir,
bad_test.path,
bad_test.path,
bad_test.path))
proc = aexpect.Expect(command=cmd_line, linesep='')
proc.read_until_last_line_matches(os.path.basename(bad_test.path))
proc.sendline('\x03')
proc.read_until_last_line_matches('Interrupt requested. Waiting 2 '
'seconds for test to finish '
'(ignoring new Ctrl+C until then)')
# We have to actually wait 2 seconds until the ignore window is over
time.sleep(2)
proc.sendline('\x03')
proc.read_until_last_line_matches('TIME : %d s')
wait.wait_for(lambda: not proc.is_alive(), timeout=1)
# Make sure the bad test will be really gone from the process table
def wait_until_no_badtest():
bad_test_processes = []
old_psutil = False
try:
process_list = psutil.pids()
except AttributeError:
process_list = psutil.get_pid_list()
old_psutil = True
for p in process_list:
try:
p_obj = psutil.Process(p)
if p_obj is not None:
if old_psutil:
cmdline_list = psutil.Process(p).cmdline
else:
cmdline_list = psutil.Process(p).cmdline()
if bad_test.path in " ".join(cmdline_list):
bad_test_processes.append(p_obj)
# psutil.NoSuchProcess happens when the original
# process already ended and left the process table
except psutil.NoSuchProcess:
pass
return len(bad_test_processes) == 0
wait.wait_for(wait_until_no_badtest, timeout=2)
# Make sure the Killing test subprocess message did appear
self.assertIn('Killing test subprocess', proc.get_output())
def wait_for(func, step=1, text=None, timeout=None, throw_exc=False, **kwargs):
"""
Wrapper function to wait with timeout option.
If timeout received, avocado 'wait_for' method will be used.
Otherwise the below function will be called.
:param func: Function to evaluate.
:param step: Time to sleep between attempts in seconds
:param text: Text to print while waiting, for debug purposes
:param timeout: Timeout in seconds
:param throw_exc: Raise exception if timeout expired, but func result is not True
:param kwargs: Keyword arguments to func
:return: Return value of func.
"""
if not timeout:
return forever_wait_for(func, step, text, **kwargs)
if kwargs:
def func_wrap():
return func(**kwargs)
res = wait.wait_for(func_wrap, timeout=timeout, step=step, text=text)
else:
res = wait.wait_for(func, timeout=timeout, step=step, text=text)
if res is not True:
err = 'Wait for: {}: timeout - {} seconds - expired'.format(text, timeout)
LOGGER.warning(err)
if throw_exc:
raise WaitTimeoutExpired(err)
return res
def test_badly_behaved(self):
"""
Make sure avocado can cleanly get out of a loop of badly behaved tests.
"""
bad_test_basename = ('wontquit-%s' %
data_factory.generate_random_string(5))
bad_test = script.TemporaryScript(bad_test_basename,
BAD_TEST,
'avocado_interrupt_test',
mode=DEFAULT_MODE)
bad_test.save()
os.chdir(basedir)
cmd_line = ('./scripts/avocado run --sysinfo=off --job-results-dir %s '
'%s %s %s' %
(self.tmpdir, bad_test.path, bad_test.path, bad_test.path))
proc = aexpect.Expect(command=cmd_line, linesep='')
proc.read_until_last_line_matches(os.path.basename(bad_test.path))
proc.sendline('\x03')
proc.read_until_last_line_matches('Interrupt requested. Waiting 2 '
'seconds for test to finish '
'(ignoring new Ctrl+C until then)')
# We have to actually wait 2 seconds until the ignore window is over
time.sleep(2.5)
proc.sendline('\x03')
proc.read_until_last_line_matches('TESTS TIME : %d s')
wait.wait_for(lambda: not proc.is_alive(), timeout=1)
# Make sure the bad test will be really gone from the process table
def wait_until_no_badtest():
bad_test_processes = []
old_psutil = False
try:
process_list = psutil.pids()
except AttributeError:
process_list = psutil.get_pid_list()
old_psutil = True
for p in process_list:
try:
p_obj = psutil.Process(p)
if p_obj is not None:
if old_psutil:
cmdline_list = psutil.Process(p).cmdline
else:
cmdline_list = psutil.Process(p).cmdline()
if bad_test.path in " ".join(cmdline_list):
bad_test_processes.append(p_obj)
# psutil.NoSuchProcess happens when the original
# process already ended and left the process table
except psutil.NoSuchProcess:
pass
return len(bad_test_processes) == 0
wait.wait_for(wait_until_no_badtest, timeout=2)
# Make sure the Killing test subprocess message did appear
self.assertIn('Killing test subprocess', proc.get_output())
def set_multipath_disks_status(disks, status):
"""
set multiple paths to same status. all disks online or offline.
multipath like below:
mpatha (360050763008084e6e0000000000001a4) dm-3 IBM,2145
size=100G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=50 status=active
| `- 1:0:1:0 sdc 8:32 active ready running
`-+- policy='service-time 0' prio=10 status=enabled
`- 1:0:0:0 sdb 8:16 failed faulty offline
:param disks: disk list.
:param status: the state of disk. online/offline
:return: by default
"""
for disk in disks:
set_disk_status_to_online_offline(disk, status)
time.sleep(2)
if len(disks) == 1:
wait.wait_for(lambda: compare_onepath_status(status, disks[0]),
first=wait_time,
step=3,
timeout=60)
else:
wait.wait_for(lambda: compare_multipath_status(status),
first=wait_time,
step=3,
timeout=60)
def wait_for(func, step=1, text=None, timeout=None, throw_exc=False, **kwargs):
"""
Wrapper function to wait with timeout option.
If timeout received, avocado 'wait_for' method will be used.
Otherwise the below function will be called.
:param func: Function to evaluate.
:param step: Time to sleep between attempts in seconds
:param text: Text to print while waiting, for debug purposes
:param timeout: Timeout in seconds
:param throw_exc: Raise exception if timeout expired, but func result is not True
:param kwargs: Keyword arguments to func
:return: Return value of func.
"""
if not timeout:
return forever_wait_for(func, step, text, **kwargs)
if kwargs:
def func_wrap():
return func(**kwargs)
res = wait.wait_for(func_wrap, timeout=timeout, step=step, text=text)
else:
res = wait.wait_for(func, timeout=timeout, step=step, text=text)
if res is not True:
err = 'Wait for: {}: timeout - {} seconds - expired'.format(
text, timeout)
LOGGER.warning(err)
if throw_exc:
raise WaitTimeoutExpired(err)
return res
def hotplug_add(self, slot, pci_addr):
"""
Hot plug add operation
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % slot, "1")
def is_added():
"""
Returns True if pci device is added, False otherwise.
"""
if pci_addr not in pci.get_pci_addresses():
return False
return True
def is_recovered():
"""
Compares current endpoint devices in pci address with
`pre` value, and returns True if equals pre.
False otherwise.
"""
post = len(pci.get_disks_in_pci_address(pci_addr))
post += len(pci.get_nics_in_pci_address(pci_addr))
self.log.debug("Pre: %d, Post: %d",
self.end_devices[pci_addr], post)
if post == self.end_devices[pci_addr]:
return True
return False
if not wait.wait_for(is_added, timeout=10):
return False
# Waiting for 10s per end device, for recovery.
return wait.wait_for(is_recovered,
self.end_devices[pci_addr] * 10)
def form_conf_mpath_file(blacklist="", defaults_extra=""):
"""
Form a multipath configuration file, and restart multipath service.
:param blacklist: Entry in conf file to indicate blacklist section.
:param defaults_extra: Extra entry in conf file in defaults section.
"""
conf_file = "/etc/multipath.conf"
with open(conf_file, "w", encoding='utf-8') as mpath_fp:
mpath_fp.write("defaults {\n")
mpath_fp.write(" find_multipaths yes\n")
mpath_fp.write(" user_friendly_names yes\n")
if defaults_extra:
mpath_fp.write(f" {defaults_extra}\n")
mpath_fp.write("}\n")
if blacklist:
mpath_fp.write("blacklist {\n")
mpath_fp.write(f" {blacklist}\n")
mpath_fp.write("}\n")
LOG.debug(open(conf_file, "r", encoding='utf-8').read())
# The reason for sleep here is to give some time for change in
# multipath.conf file to take effect.
time.sleep(5)
mpath_svc = service.SpecificServiceManager(get_svc_name())
mpath_svc.restart()
wait.wait_for(mpath_svc.status, timeout=10)
def test_well_behaved(self):
"""
Make sure avocado can cleanly get out of a loop of well behaved tests.
"""
good_test_basename = ('goodtest-%s.py' %
data_factory.generate_random_string(5))
good_test = script.TemporaryScript(good_test_basename,
GOOD_TEST,
'avocado_interrupt_test',
mode=DEFAULT_MODE)
good_test.save()
os.chdir(basedir)
cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
'%s %s %s' % (AVOCADO, self.tmpdir, good_test.path,
good_test.path, good_test.path))
proc = aexpect.Expect(command=cmd_line, linesep='')
proc.read_until_last_line_matches(os.path.basename(good_test.path))
proc.sendline('\x03')
proc.read_until_last_line_matches('TESTS TIME : %d s')
wait.wait_for(lambda: not proc.is_alive(), timeout=1)
# Make sure the good test will be really gone from the process table
def wait_until_no_goodtest():
good_test_processes = []
old_psutil = False
try:
process_list = psutil.pids()
except AttributeError:
process_list = psutil.get_pid_list()
old_psutil = True
for p in process_list:
try:
p_obj = psutil.Process(p)
if p_obj is not None:
if old_psutil:
cmdline_list = psutil.Process(p).cmdline
else:
try:
cmdline_list = psutil.Process(p).cmdline()
except psutil.AccessDenied:
cmdline_list = []
if good_test.path in " ".join(cmdline_list):
good_test_processes.append(p_obj)
# psutil.NoSuchProcess happens when the original
# process already ended and left the process table
except psutil.NoSuchProcess:
pass
return len(good_test_processes) == 0
wait.wait_for(wait_until_no_goodtest, timeout=2)
# Make sure the Killing test subprocess message is not there
self.assertNotIn('Killing test subprocess', proc.get_output())
# Make sure the Interrupted requested sentence is there
self.assertIn(
'Interrupt requested. Waiting 2 seconds for test to '
'finish (ignoring new Ctrl+C until then)', proc.get_output())
def disrupt_abort_repair(self):
"""
Start repair target_node in background, then try to abort the repair streaming.
"""
self._set_current_disruption('AbortRepairMonkey')
self.log.debug("Start repair target_node in background")
thread1 = threading.Thread(target=self.repair_nodetool_repair)
thread1.start()
def repair_streaming_exists():
result = self.target_node.remoter.run(
'curl -X GET --header "Content-Type: application/json" --header "Accept: application/json" "http://127.0.0.1:10000/stream_manager/stream_manager/"'
)
return 'repair-' in result.stdout
wait.wait_for(func=repair_streaming_exists,
timeout=10,
step=0.01,
text='Wait for repair starts')
self.log.debug(
"Abort repair streaming by storage_service/force_terminate_repair API"
)
self.target_node.remoter.run(
'curl -X GET --header "Content-Type: application/json" --header "Accept: application/json" "http://127.0.0.1:10000/storage_service/force_terminate_repair"'
)
thread1.join(timeout=120)
self.log.debug("Execute a complete repair for target node")
self.repair_nodetool_repair()
def upgrade(self, name=None):
"""
Upgrade all packages of the system with eventual new versions.
Optionally, upgrade individual packages.
:param name: optional parameter wildcard spec to upgrade
:type name: str
"""
def update_pkg_list():
ud_command = 'update'
ud_cmd = self.base_command + ' ' + ud_command
try:
process.system(ud_cmd)
return True
except process.CmdError:
return False
wait.wait_for(update_pkg_list, timeout=300, step=30,
text="Wait until package list is up to date...")
if name:
up_command = 'install --only-upgrade'
up_cmd = " ".join([self.base_command, self.dpkg_force_confdef,
up_command, name])
else:
up_command = 'upgrade'
up_cmd = " ".join([self.base_command, self.dpkg_force_confdef,
up_command])
try:
process.system(up_cmd, shell=True, sudo=True)
return True
except process.CmdError:
return False
def test_well_behaved(self):
"""
Make sure avocado can cleanly get out of a loop of well behaved tests.
"""
good_test_basename = ('goodtest-%s.py' %
data_factory.generate_random_string(5))
good_test = script.TemporaryScript(good_test_basename, GOOD_TEST,
'avocado_interrupt_test',
mode=DEFAULT_MODE)
good_test.save()
os.chdir(basedir)
cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
'%s %s %s' % (AVOCADO, self.tmpdir, good_test.path,
good_test.path, good_test.path))
proc = aexpect.Expect(command=cmd_line, linesep='')
proc.read_until_last_line_matches(os.path.basename(good_test.path))
proc.sendline('\x03')
proc.read_until_last_line_matches('JOB TIME : %d s')
wait.wait_for(lambda: not proc.is_alive(), timeout=1)
# Make sure the good test will be really gone from the process table
def wait_until_no_goodtest():
good_test_processes = []
old_psutil = False
try:
process_list = psutil.pids()
except AttributeError:
process_list = psutil.get_pid_list()
old_psutil = True
for p in process_list:
try:
p_obj = psutil.Process(p)
if p_obj is not None:
if old_psutil:
cmdline_list = psutil.Process(p).cmdline
else:
try:
cmdline_list = psutil.Process(p).cmdline()
except psutil.AccessDenied:
cmdline_list = []
if good_test.path in " ".join(cmdline_list):
good_test_processes.append(p_obj)
# psutil.NoSuchProcess happens when the original
# process already ended and left the process table
except psutil.NoSuchProcess:
pass
return len(good_test_processes) == 0
wait.wait_for(wait_until_no_goodtest, timeout=2)
# Make sure the Killing test subprocess message is not there
self.assertNotIn('Killing test subprocess', proc.get_output())
# Make sure the Interrupted requested sentence is there
self.assertIn('Interrupt requested. Waiting 2 seconds for test to '
'finish (ignoring new Ctrl+C until then)',
proc.get_output())
def setUp(self):
"""
configure the virtual machine that will be used in the tests
"""
self.cmd_run = self.params.get('command', default=None)
self.debug = self.params.get('debug', default=False)
self.iso_src = self.params.get('minishift_iso_src_path',
default='~/.contra-env-setup/minishift')
self.iso_dest = self.params.get(
'minishift_iso_dest_path', default='~/.contra-env-setup/minishift')
self.username = self.params.get('username', default='avocado')
self.password = self.params.get('password', default='avocado')
self.vm_name = self.params.get('vm_name',
default='contra-env-setup-01')
self.extra_cmd_run = self.params.get('extra_command', default=None)
self.distro = self.params.get('distro', default='Fedora')
self.pkg_mgm = self.params.get('pkg_mgm', default='dnf')
self.os_variant = self.params.get('os_variant', default='fedora27')
project = 'https://github.com/CentOS-PaaS-SIG/contra-env-setup.git'
repo = self.params.get('repo', default=project)
branch = self.params.get('branch', default='master')
git.get_repo(repo, branch=branch, destination_dir=self.workdir)
self.image = vmimage.get(self.distro,
cache_dir='~/avocado/data/cache/')
process.run('{qemu} resize '
'{image_path} 40G'.format(qemu=_QEMU_BIN,
image_path=self.image.path))
cmd = [
_VIRTINST_BIN, '--import', '--noautoconsole', '--connect',
'qemu:///system', '--name', self.vm_name, '--disk',
'%s,size=40' % self.image.path, '--graphics', 'vnc', '--cpu',
'host', '--memory', '7168', '--vcpus', '4', '--os-variant',
self.os_variant, '--disk',
'%s,device=cdrom,format=iso' %
create_cloudinit_cdrom(self.vm_name, self.username, self.password)
]
process.run(' '.join(cmd))
wait.wait_for(ip_available, timeout=60, args=[self.vm_name])
cmd = [
_VIRSH_BIN, '--connect', 'qemu:///system', 'domifaddr',
self.vm_name
]
result = process.run(' '.join(cmd))
for line in result.stdout_text.splitlines():
if 'ipv4' in line:
self.vm_ip = line.split()[3].split('/')[0]
wait.wait_for(ssh_available,
timeout=60,
args=[self.vm_ip, self.username])
def delete_fs(self, l_disk):
"""
checks for disk/dir mount, unmount if mounted and checks for
filesystem exitance and wipe it off after dir/disk unmount.
:param l_disk: disk name for which you want to check the mount status
:return: None
"""
def is_fs_deleted():
cmd = "wipefs -af %s" % l_disk
process.system(cmd, shell=True, ignore_status=True)
if disk.fs_exists(l_disk):
return False
return True
def is_disk_unmounted():
cmd = "umount %s" % l_disk
cmd1 = 'umount /dev/mapper/avocado_vg-avocado_lv'
process.system(cmd, shell=True, ignore_status=True)
process.system(cmd1, shell=True, ignore_status=True)
if disk.is_disk_mounted(l_disk):
return False
return True
def is_dir_unmounted():
cmd = 'umount %s' % self.dir
process.system(cmd, shell=True, ignore_status=True)
if disk.is_dir_mounted(self.dir):
return False
return True
self.log.info("checking if disk is mounted.")
if disk.is_disk_mounted(l_disk):
self.log.info("%s is mounted, unmounting it ....", l_disk)
if wait.wait_for(is_disk_unmounted, timeout=10):
self.log.info("%s unmounted successfully" % l_disk)
else:
self.err_mesg.append("%s unmount failed", l_disk)
else:
self.log.info("disk %s not mounted." % l_disk)
self.log.info("checking if dir %s is mounted." % self.dir)
if disk.is_dir_mounted(self.dir):
self.log.info("%s is mounted, unmounting it ....", self.dir)
if wait.wait_for(is_dir_unmounted, timeout=10):
self.log.info("%s unmounted successfully" % self.dir)
else:
self.err_mesg.append("failed to unount %s", self.dir)
else:
self.log.info("dir %s not mounted." % self.dir)
self.log.info("checking if fs exists in {}" .format(l_disk))
if disk.fs_exists(l_disk):
self.log.info("found fs on %s, removing it....", l_disk)
if wait.wait_for(is_fs_deleted, timeout=10):
self.log.info("fs removed successfully..")
else:
self.err_mesg.append(f'failed to delete fs on {l_disk}')
else:
self.log.info(f'No fs detected on {self.disk}')
self.log.info("Running dd...")
def run_background_test(self):
"""Run virt sub test in backgroup"""
wait_time = float(self.params.get("sub_test_wait_time", 0))
args = (self.test, self.params, self.env, self.params["sub_type"])
bg_test = BackgroundTest(run_virt_sub_test, args)
bg_test.start()
wait_for(bg_test.is_alive, first=wait_time, step=3, timeout=240)
return bg_test
def run_background_test(self):
"""Run virt sub test in backgroup"""
wait_time = float(self.params.get("sub_test_wait_time", 0))
args = (self.test, self.params, self.env, self.params["sub_type"])
bg_test = BackgroundTest(run_virt_sub_test, args)
bg_test.start()
wait_for(bg_test.is_alive, first=wait_time, step=3, timeout=240)
return bg_test
def tearDown(self):
"""
Restore config file, if existed, and restart services
"""
if os.path.isfile(self.mpath_file):
shutil.copyfile("%s.bkp" % self.mpath_file, self.mpath_file)
self.mpath_svc.restart()
# Need to wait for some time to make sure multipaths are loaded.
wait.wait_for(self.mpath_svc.status, timeout=10)
def tearDown(self):
"""
Restore config file, if existed, and restart services
"""
if os.path.isfile(self.mpath_file):
shutil.copyfile("%s.bkp" % self.mpath_file, self.mpath_file)
self.mpath_svc.restart()
# Need to wait for some time to make sure multipaths are loaded.
wait.wait_for(self.mpath_svc.status, timeout=10)
def close(self):
"""
Safely postprocess the container
:note: It won't remove the container, you need to do it manually
"""
if self._docker:
self._docker.sendline("exit")
# Leave the process up to 10s to finish, then nuke it
wait_for(lambda: not self._docker.is_alive(), 10)
self._docker.close()
def close(self):
"""
Safely postprocess the container
:note: It won't remove the container, you need to do it manually
"""
if self._docker:
self._docker.sendline("exit")
# Leave the process up to 10s to finish, then nuke it
wait_for(lambda: not self._docker.is_alive(), 10)
self._docker.close()
def disrupt_nodetool_enospc(self, sleep_time=30, all_nodes=False):
if all_nodes:
nodes = self.cluster.nodes
else:
nodes = [self.target_node]
self._set_current_disruption('Enospc test on {}'.format([n.name for n in nodes]))
def search_database_enospc(node):
"""
Search database log by executing cmd inside node, use shell tool to
avoid return and process huge data.
"""
cmd = "journalctl --no-tail --no-pager -u scylla-server.service|grep 'No space left on device'|wc -l"
result = node.remoter.run(cmd, verbose=True)
return int(result.stdout)
def approach_enospc():
# get the size of free space (default unit: KB)
result = node.remoter.run("df -l|grep '/var/lib/scylla'")
free_space_size = result.stdout.split()[3]
occupy_space_size = int(int(free_space_size) * 90 / 100)
occupy_space_cmd = 'sudo fallocate -l {}K /var/lib/scylla/occupy_90percent.{}'.format(occupy_space_size, datetime.datetime.now().strftime('%s'))
self.log.debug('Cost 90% free space on /var/lib/scylla/ by {}'.format(occupy_space_cmd))
try:
node.remoter.run(occupy_space_cmd, verbose=True)
except Exception as details:
self.log.error(str(details))
return search_database_enospc(node) > orig_errors
for node in nodes:
result = node.remoter.run('cat /proc/mounts')
if '/var/lib/scylla' not in result.stdout:
self.log.error("Scylla doesn't use an individual storage, skip enospc test")
continue
# check original ENOSPC error
orig_errors = search_database_enospc(node)
wait.wait_for(func=approach_enospc,
timeout=300,
step=5,
text='Wait for new ENOSPC error occurs in database')
self.log.debug('Sleep {} seconds before releasing space to scylla'.format(sleep_time))
time.sleep(sleep_time)
self.log.debug('Delete occupy_90percent file to release space to scylla-server')
node.remoter.run('sudo rm -rf /var/lib/scylla/occupy_90percent.*')
self.log.debug('Sleep a while before restart scylla-server')
time.sleep(sleep_time / 2)
node.remoter.run('sudo systemctl restart scylla-server.service')
node.wait_db_up()
def assert_migration(self, src_vm, dst_vm):
wait.wait_for(self.migration_finished,
timeout=self.timeout,
step=0.1,
args=(src_vm, ))
self.assertEqual(
src_vm.command('query-migrate')['status'], 'completed')
self.assertEqual(
dst_vm.command('query-migrate')['status'], 'completed')
self.assertEqual(dst_vm.command('query-status')['status'], 'running')
self.assertEqual(
src_vm.command('query-status')['status'], 'postmigrate')
def disrupt_nodetool_enospc(self, sleep_time=30, all_nodes=False):
if all_nodes:
nodes = self.cluster.nodes
else:
nodes = [self.target_node]
self._set_current_disruption('Enospc test on {}'.format([n.name for n in nodes]))
def search_database_enospc(node):
"""
Search database log by executing cmd inside node, use shell tool to
avoid return and process huge data.
"""
cmd = "journalctl --no-tail --no-pager -u scylla-server.service|grep 'No space left on device'|wc -l"
result = node.remoter.run(cmd, verbose=True)
return int(result.stdout)
def approach_enospc():
# get the size of free space (default unit: KB)
result = node.remoter.run("df -l|grep '/var/lib/scylla'")
free_space_size = result.stdout.split()[3]
occupy_space_size = int(int(free_space_size) * 90 / 100)
occupy_space_cmd = 'sudo fallocate -l {}K /var/lib/scylla/occupy_90percent.{}'.format(occupy_space_size, datetime.datetime.now().strftime('%s'))
self.log.debug('Cost 90% free space on /var/lib/scylla/ by {}'.format(occupy_space_cmd))
try:
node.remoter.run(occupy_space_cmd, verbose=True)
except Exception as details:
self.log.error(str(details))
return search_database_enospc(node) > orig_errors
for node in nodes:
result = node.remoter.run('cat /proc/mounts')
if '/var/lib/scylla' not in result.stdout:
self.log.error("Scylla doesn't use an individual storage, skip enospc test")
continue
# check original ENOSPC error
orig_errors = search_database_enospc(node)
wait.wait_for(func=approach_enospc,
timeout=300,
step=5,
text='Wait for new ENOSPC error occurs in database')
self.log.debug('Sleep {} seconds before releasing space to scylla'.format(sleep_time))
time.sleep(sleep_time)
self.log.debug('Delete occupy_90percent file to release space to scylla-server')
node.remoter.run('sudo rm -rf /var/lib/scylla/occupy_90percent.*')
self.log.debug('Sleep a while before restart scylla-server')
time.sleep(sleep_time / 2)
node.remoter.run('sudo systemctl restart scylla-server.service')
node.wait_db_up()
def try_report_uuid(self):
uuid_path = '/var/lib/scylla-housekeeping/housekeeping.uuid'
mark_path = '/var/lib/scylla-housekeeping/housekeeping.uuid.marked'
cmd = 'curl "https://i6a5h9l1kl.execute-api.us-east-1.amazonaws.com/prod/check_version?uu=%s&mark=scylla"'
wait.wait_for(lambda: os.path.exists(uuid_path), timeout=30, step=5,
text='Waiting for housekeeping.uuid generated')
if os.path.exists(uuid_path) and not os.path.exists(mark_path):
with open(uuid_path) as uuid_file:
uuid = uuid_file.read().strip()
self.log.debug('housekeeping.uuid is %s', uuid)
process.run(cmd % uuid, shell=True, verbose=True)
process.run('sudo -u scylla touch %s' % mark_path, verbose=True)
def test_disable_enable_dev(self):
'''
Test if disabling and enabling of an adapter works
'''
self.disable_enable_dev('d')
self.is_disabled_enabled_dev('1')
self.disable_enable_dev('e')
self.is_disabled_enabled_dev('0')
device = self.find_device(self.mac_id[0])
networkinterface = NetworkInterface(device, self.local)
wait.wait_for(networkinterface.is_link_up, timeout=60)
if networkinterface.ping_check(self.peer_ip[0], count=5) is not None:
self.fail("Enabling and disabling of the interface has affected network connectivity")
self.check_dmesg_error()
def test_badly_behaved_sigint(self):
"""
Make sure avocado can cleanly get out of a loop of badly behaved tests.
:avocado: tags=parallel:1
"""
bad_test_basename = ('wontquit-%s' %
data_factory.generate_random_string(5))
bad_test = script.TemporaryScript(bad_test_basename,
BAD_TEST,
'avocado_interrupt_test',
mode=DEFAULT_MODE)
bad_test.save()
self.test_module = bad_test.path
os.chdir(BASEDIR)
cmd = ('%s run %s --sysinfo=off --job-results-dir %s ' %
(AVOCADO, self.test_module, self.tmpdir.name))
proc = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if not wait.wait_for(lambda: self.has_children(proc), timeout=10):
process.kill_process_tree(proc.pid)
self.fail('Avocado did not start the test process.')
# This test will ignore SIGINT, so it should terminate
# when we send the second SIGINT.
os.kill(proc.pid, signal.SIGINT)
# We have to actually wait 2+ seconds until
# the ignore window is over
time.sleep(2.5)
os.kill(proc.pid, signal.SIGINT)
if not wait.wait_for(lambda: self.is_finished(proc), timeout=30):
process.kill_process_tree(proc.pid)
self.fail('Avocado was still running after receiving SIGINT '
'twice.')
self.assertTrue(
wait.wait_for(self._no_test_in_process_table, timeout=10),
'Avocado left processes behind.')
output = proc.stdout.read()
# Make sure the Interrupted requested sentence is there
self.assertIn(
b'Interrupt requested. Waiting 2 seconds for test to '
b'finish (ignoring new Ctrl+C until then)', output)
# Make sure the Killing test subprocess message did appear
self.assertIn(b'Killing test subprocess', output)
def rmc_service_start(self, server):
'''
Start RMC services which is needed for LPM migration
'''
for svc in ["-z", "-A", "-p"]:
process.run('/opt/rsct/bin/rmcctrl %s' % svc,
shell=True,
sudo=True)
if not wait.wait_for(self.is_RMC_active(server), timeout=60):
process.run('/usr/sbin/rsct/install/bin/recfgct',
shell=True,
sudo=True)
process.run('/opt/rsct/bin/rmcctrl -p', shell=True, sudo=True)
if not wait.wait_for(self.is_RMC_active(server), timeout=300):
self.fail("ERROR : RMC connection is down !!")
def _check_install_key_required(self):
def _safe_ssh_ping():
try:
self._ssh_ping()
return True
except (SSHPermissionDeniedError, process.CmdError):
return None
if not self.key_file and self.password:
try:
self._ssh_ping()
except (SSHPermissionDeniedError, process.CmdError):
copy_id_cmd = ('ssh-copy-id -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p %s %s@%s' %
(self.port, self.user, self.hostname))
while True:
try:
expect = aexpect.Expect(copy_id_cmd)
expect.read_until_output_matches(['.*password:'******'Waiting for password-less SSH')
if result is None:
raise SSHPermissionDeniedError('Unable to configure '
'password less SSH. '
'Output of %s: %s' %
(copy_id_cmd,
expect.get_output()))
else:
self.log.info('Successfully configured SSH key auth')
def module_load_unload(self, mod1):
"""
Unloading and loading the given module
"""
if linux_modules.module_is_loaded(mod1) is False:
linux_modules.load_module(mod1)
time.sleep(self.load_unload_sleep_time)
sub_mod = linux_modules.get_submodules(mod1)
if sub_mod:
for mod in sub_mod:
linux_modules.unload_module(mod)
if linux_modules.module_is_loaded(mod) is True:
self.error_modules.append(mod)
break
if linux_modules.unload_module(mod1) is False:
self.fail("Unloading Module %s failed" % mod1)
time.sleep(self.load_unload_sleep_time)
cmd = "%s %s=%s" % (mod1, self.param_name, self.param_value)
if linux_modules.load_module(cmd) is False:
self.fail("Param %s = Value %s Failed for Module %s" %
(self.param_name, self.param_value, mod1))
time.sleep(self.load_unload_sleep_time)
if self.sysfs_chk:
if self.sysfs_value_check() is False:
self.fail("Sysfs check failed ")
if not wait.wait_for(configure_network.is_interface_link_up,
timeout=120,
args=[self.ifaces]):
self.fail("Link up of interface is taking longer than 120s")
if not configure_network.ping_check(
self.ifaces, self.peer, '1000', flood=True):
self.fail("ping test failed")
def test_vnic_dlpar(self):
'''
Perform vNIC device hot add and hot remove using drmgr command
'''
for slot_no, device_ip, netmask, mac, peer_ip in zip(
self.slot_num, self.device_ip, self.netmask, self.mac_id,
self.peer_ip):
self.update_backing_devices(slot_no)
dev_id = self.find_device_id(mac)
device_name = self.find_device(mac)
slot = self.find_virtual_slot(dev_id)
if slot:
try:
for _ in range(self.num_of_dlpar):
self.drmgr_vnic_dlpar('-r', slot)
self.drmgr_vnic_dlpar('-a', slot)
self.wait_intrerface(device_name)
except CmdError as details:
self.log.debug(str(details))
self.fail("dlpar operation did not complete")
device = self.find_device(mac)
networkinterface = NetworkInterface(device, self.local)
try:
networkinterface.add_ipaddr(device_ip, netmask)
except Exception:
networkinterface.save(device_ip, netmask)
if not wait.wait_for(networkinterface.is_link_up, timeout=120):
self.fail("Unable to bring up the link on the Network \
virtualized device")
if networkinterface.ping_check(peer_ip, count=5) is not None:
self.fail("dlpar has affected Network connectivity")
else:
self.fail("slot not found")
self.check_dmesg_error()
def setUp(self):
"""
Set up.
"""
self.iface = self.params.get("interface", default="")
self.count = self.params.get("count", default="500")
self.nping_count = self.params.get("nping_count", default="")
self.peer_ip = self.params.get("peer_ip", default="")
self.drop = self.params.get("drop_accepted", default="10")
self.host_ip = self.params.get("host_ip", default="")
self.option = self.params.get("option", default='')
# Check if interface exists in the system
interfaces = netifaces.interfaces()
if self.iface not in interfaces:
self.cancel("%s interface is not available" % self.iface)
if not self.peer_ip:
self.cancel("peer ip should specify in input")
self.ipaddr = self.params.get("host_ip", default="")
self.netmask = self.params.get("netmask", default="")
configure_network.set_ip(self.ipaddr, self.netmask, self.iface)
if not wait.wait_for(configure_network.is_interface_link_up,
timeout=120,
args=[self.iface]):
self.cancel(
"Link up of interface is taking longer than 120 seconds")
self.peer_user = self.params.get("peer_user", default="root")
self.peer_password = self.params.get("peer_password",
'*',
default="None")
self.mtu = self.params.get("mtu", default=1500)
self.peerinfo = PeerInfo(self.peer_ip,
peer_user=self.peer_user,
peer_password=self.peer_password)
self.peer_interface = self.peerinfo.get_peer_interface(self.peer_ip)
if not self.peerinfo.set_mtu_peer(self.peer_interface, self.mtu):
self.cancel("Failed to set mtu in peer")
if not configure_network.set_mtu_host(self.iface, self.mtu):
self.cancel("Failed to set mtu in host")
# Install needed packages
smm = SoftwareManager()
detected_distro = distro.detect()
pkgs = ['tcpdump', 'flex', 'bison', 'gcc', 'gcc-c++', 'nmap']
for pkg in pkgs:
if not smm.check_installed(pkg) and not smm.install(pkg):
self.cancel("%s package Can not install" % pkg)
if detected_distro.name == "SuSE":
self.nmap = os.path.join(self.teststmpdir, 'nmap')
nmap_download = self.params.get("nmap_download",
default="https:"
"//nmap.org/dist/"
"nmap-7.80.tar.bz2")
tarball = self.fetch_asset(nmap_download)
self.version = os.path.basename(tarball.split('.tar')[0])
self.n_map = os.path.join(self.nmap, self.version)
archive.extract(tarball, self.nmap)
os.chdir(self.n_map)
process.system('./configure ppc64le', shell=True)
build.make(self.n_map)
process.system('./nping/nping -h', shell=True)
def run_process_to_use_mnt(self):
proc = process.SubProcess(self.use_mnt_cmd, sudo=True)
proc.start()
self.assertTrue(wait.wait_for(lambda: os.path.exists(self.use_mnt_file),
timeout=1, first=0.1, step=0.1),
"File was not created within mountpoint")
return proc
def run_process_to_use_mnt(self):
proc = process.SubProcess(self.use_mnt_cmd, sudo=True)
proc.start()
self.assertTrue(wait.wait_for(lambda: os.path.exists(self.use_mnt_file),
timeout=1, first=0.1, step=0.1),
"File was not created within mountpoint")
return proc
def is_recovered():
"""
Checks if the block device adapter is recovers all its disks/paths
properly after hotplug of adapter.
Returns True if all disks/paths back online after adapter added
Back, else False.
"""
def is_path_online():
path_stat = list(multipath.get_path_status(curr_path))
if path_stat[0] != 'active' or path_stat[2] != 'ready':
return False
return True
curr_path = ''
err_disks = []
if pci.get_pci_class_name(pci_addr) == 'fc_host':
disks = pci.get_disks_in_pci_address(pci_addr)
for disk in disks:
curr_path = disk.split("/")[-1]
self.log.info("curr_path=%s" % curr_path)
if not wait.wait_for(is_path_online, timeout=10):
self.log.info("%s failed to recover after add" % disk)
err_disks.append(disk)
if err_disks:
self.log.info("few paths failed to recover : %s" % err_disks)
return False
return True
def test_add(self):
'''
Network virtualized device add operation
'''
for slot, mac, sriov_port, adapter_id, device_ip, netmask in zip(
self.slot_num, self.mac_id, self.sriov_port,
self.backing_adapter_id, self.device_ip, self.netmask):
if not self.check_slot_availability(slot):
self.fail("Slot does not exist")
self.device_add_remove(slot, mac, sriov_port, adapter_id, 'add')
self.interface_naming(mac, slot)
output = self.list_device(slot)
if 'slot_num=%s' % slot not in str(output):
self.log.debug(output)
self.fail("lshwres fails to list Network virtualized device \
after add operation")
if mac not in str(output):
self.log.debug(output)
self.fail("MAC address in HMC differs")
if not self.find_device(mac):
self.fail("MAC address differs in linux")
device = self.find_device(mac)
networkinterface = NetworkInterface(device, self.local)
try:
networkinterface.add_ipaddr(device_ip, netmask)
networkinterface.save(device_ip, netmask)
except Exception:
networkinterface.save(device_ip, netmask)
networkinterface.bring_up()
if not wait.wait_for(networkinterface.is_link_up, timeout=120):
self.fail("Unable to bring up the link on the Network \
virtualized device")
self.check_dmesg_error()
def wait_services_up(self):
service_start_timeout = 900
output = wait.wait_for(func=self._scylla_service_is_up,
timeout=service_start_timeout, step=5)
if output is None:
e_msg = 'Scylla service does not appear to be up after %s s' % service_start_timeout
raise StartServiceError(e_msg)
def delete_raid(self):
"""
it checks for existing of raid and deletes it if exists
"""
self.log.info("deleting Sraid %s" % self.raid_name)
def is_raid_deleted():
self.sw_raid.stop()
self.sw_raid.clear_superblock()
self.log.info("checking for raid metadata")
cmd = "wipefs -af %s" % self.disk
process.system(cmd, shell=True, ignore_status=True)
if self.sw_raid.exists():
return False
return True
self.log.info("checking lvm_metadata on %s" % self.raid_name)
cmd = 'blkid -o value -s TYPE %s' % self.raid_name
out = process.system_output(cmd, shell=True,
ignore_status=True).decode("utf-8")
if out == 'LVM2_member':
cmd = "wipefs -af %s" % self.raid_name
process.system(cmd, shell=True, ignore_status=True)
if wait.wait_for(is_raid_deleted, timeout=10):
self.log.info("software raid %s deleted" % self.raid_name)
else:
self.err_mesg.append("failed to delete sraid %s" % self.raid_name)
def mpath_state_check(self, ports, state1, state2):
'''
checking mpath disk status after disabling the switch port
'''
curr_path = ''
err_paths = []
def is_path_online():
path_stat = multipath.get_path_status(curr_path)
if path_stat[0] != state1 or path_stat[2] != state2:
return False
return True
for port in ports:
paths = self.get_paths(self.dic[port])
self.log.info("verify %s path status for port %s in %s", state1,
port, ports)
for path in paths:
curr_path = path
if not wait.wait_for(is_path_online, timeout=10):
err_paths.append("%s:%s" % (port, curr_path))
if err_paths:
self.error("following paths not %s: %s" % (state1, err_paths))
else:
self.log.info("%s path verification is success", state1)
def migrate(self, protocol='tcp'):
def migrate_complete():
mig_info = self.qmp("query-migrate")
mig_status = mig_info['return']['status']
if mig_status == 'completed':
self.log("Migration successful")
return True
elif mig_status == 'failed':
raise exceptions.TestFail("Migration of %s failed" % self)
return False
clone_params = copy.copy(self.params)
clone = self.clone(params=clone_params, preserve_uuid=True)
migration_port = clone.devices.add_incoming(protocol)
self._screendump_thread_terminate(migrate=True)
clone.power_on()
uri = "%s:localhost:%d" % (protocol, migration_port)
self.qmp("migrate", uri=uri)
migrate_timeout = self.params.get('timeout',
'/plugins/virt/qemu/migrate/*')
migrate_result = wait.wait_for(migrate_complete, timeout=float(migrate_timeout),
text='Waiting for migration to complete')
if migrate_result is None:
raise exceptions.TestFail("Migration of %s did not complete after %s s" %
(self, migrate_timeout))
old_vm = VM()
old_vm.__dict__ = self.__dict__
self.__dict__ = clone.__dict__
old_vm.power_off(migrate=True)
def migrate(self, protocol='tcp'):
def migrate_complete():
mig_info = self.qmp("query-migrate")
mig_status = mig_info['return']['status']
if mig_status == 'completed':
self.log("Migration successful")
return True
elif mig_status == 'failed':
raise exceptions.TestFail("Migration of %s failed" % self)
return False
clone_params = self.params.copy()
clone_params['qemu_bin'] = path.get_qemu_dst_binary(clone_params)
clone = self.clone(params=clone_params, preserve_uuid=True)
migration_port = clone.devices.add_incoming(protocol)
self._screendump_thread_terminate(migrate=True)
clone.power_on()
uri = "%s:localhost:%d" % (protocol, migration_port)
self.qmp("migrate", uri=uri)
migrate_timeout = self.params.get('avocado.args.run.migrate.timeout',
defaults.migrate_timeout)
migrate_result = wait.wait_for(
migrate_complete,
timeout=migrate_timeout,
text='Waiting for migration to complete')
if migrate_result is None:
raise exceptions.TestFail(
"Migration of %s did not complete after %s s" %
(self, migrate_timeout))
old_vm = VM()
old_vm.__dict__ = self.__dict__
self.__dict__ = clone.__dict__
old_vm.power_off(migrate=True)
def _check_install_key_required(self):
def _safe_ssh_ping():
try:
self._ssh_ping()
return True
except (SSHPermissionDeniedError, process.CmdError):
return None
if not self.key_file and self.password:
try:
self._ssh_ping()
except (SSHPermissionDeniedError, process.CmdError):
copy_id_cmd = ('ssh-copy-id -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p %s %s@%s' %
(self.port, self.user, self.hostname))
while True:
try:
expect = aexpect.Expect(copy_id_cmd)
expect.read_until_output_matches(['.*password:'******'Waiting for password-less SSH')
if result is None:
raise SSHPermissionDeniedError('Unable to configure '
'password less SSH. '
'Output of %s: %s' %
(copy_id_cmd,
expect.get_output()))
else:
self.log.info('Successfully configured SSH key auth')
def test_migration_with_tcp_localhost(self):
source_vm = self.get_vm()
dest_uri = 'tcp:localhost:%u' % self._get_free_port()
dest_vm = self.get_vm('-incoming', dest_uri)
dest_vm.launch()
source_vm.launch()
source_vm.qmp('migrate', uri=dest_uri)
wait.wait_for(
self.migration_finished,
timeout=self.timeout,
step=0.1,
args=(source_vm,)
)
self.assertEqual(dest_vm.command('query-migrate')['status'], 'completed')
self.assertEqual(source_vm.command('query-migrate')['status'], 'completed')
self.assertEqual(dest_vm.command('query-status')['status'], 'running')
self.assertEqual(source_vm.command('query-status')['status'], 'postmigrate')
def test_badly_behaved_sigint(self):
"""
Make sure avocado can cleanly get out of a loop of badly behaved tests.
"""
bad_test_basename = ('wontquit-%s' %
data_factory.generate_random_string(5))
bad_test = script.TemporaryScript(bad_test_basename, BAD_TEST,
'avocado_interrupt_test',
mode=DEFAULT_MODE)
bad_test.save()
self.test_module = bad_test.path
os.chdir(BASEDIR)
cmd = ('%s run %s --sysinfo=off --job-results-dir %s ' %
(AVOCADO, self.test_module, self.tmpdir))
self.proc = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if not wait.wait_for(self._has_children, timeout=10):
process.kill_process_tree(self.proc.pid)
self.fail('Avocado did not start the test process.')
# This test will ignore SIGINT, so it should terminate
# when we send the second SIGINT.
os.kill(self.proc.pid, signal.SIGINT)
# We have to actually wait 2+ seconds until
# the ignore window is over
time.sleep(2.5)
os.kill(self.proc.pid, signal.SIGINT)
if not wait.wait_for(self._is_finished, timeout=10):
process.kill_process_tree(self.proc.pid)
self.fail('Avocado was still running after receiving SIGINT '
'twice.')
self.assertTrue(wait.wait_for(self._no_test_in_process_table,
timeout=10), 'Avocado left processes behind.')
output = self.proc.stdout.read()
# Make sure the Interrupted requested sentence is there
self.assertIn(b'Interrupt requested. Waiting 2 seconds for test to '
b'finish (ignoring new Ctrl+C until then)', output)
# Make sure the Killing test subprocess message did appear
self.assertIn(b'Killing test subprocess', output)
def test_badly_behaved(self):
"""
Make sure avocado can cleanly get out of a loop of badly behaved tests.
"""
bad_test_basename = "wontquit-%s" % data_factory.generate_random_string(5)
bad_test = script.TemporaryScript(bad_test_basename, BAD_TEST, "avocado_interrupt_test", mode=0755)
bad_test.save()
os.chdir(basedir)
cmd_line = "./scripts/avocado run --sysinfo=off --job-results-dir %s " "%s %s %s" % (
self.tmpdir,
bad_test.path,
bad_test.path,
bad_test.path,
)
proc = aexpect.Expect(command=cmd_line, linesep="")
proc.read_until_last_line_matches(os.path.basename(bad_test.path))
proc.sendline("\x03")
proc.read_until_last_line_matches(
"Interrupt requested. Waiting 2 " "seconds for test to finish " "(ignoring new Ctrl+C until then)"
)
# We have to actually wait 2 seconds until the ignore window is over
time.sleep(2)
proc.sendline("\x03")
proc.read_until_last_line_matches("TIME : %d s")
wait.wait_for(lambda: not proc.is_alive(), timeout=1)
# Make sure the bad test will be really gone from the process table
def wait_until_no_badtest():
bad_test_processes = []
for p in psutil.pids():
p_obj = None
try:
p_obj = psutil.Process(p)
except psutil.NoSuchProcess:
pass
if p_obj is not None:
if bad_test.path in " ".join(psutil.Process(p).cmdline()):
bad_test_processes.append(p_obj)
return len(bad_test_processes) == 0
wait.wait_for(wait_until_no_badtest, timeout=2)
# Make sure the Killing test subprocess message did appear
self.assertIn("Killing test subprocess", proc.get_output())
def test_well_behaved(self):
"""
Make sure avocado can cleanly get out of a loop of well behaved tests.
"""
good_test_basename = "goodtest-%s.py" % data_factory.generate_random_string(5)
good_test = script.TemporaryScript(good_test_basename, GOOD_TEST, "avocado_interrupt_test", mode=0755)
good_test.save()
os.chdir(basedir)
cmd_line = "./scripts/avocado run --sysinfo=off --job-results-dir %s " "%s %s %s" % (
self.tmpdir,
good_test.path,
good_test.path,
good_test.path,
)
proc = aexpect.Expect(command=cmd_line, linesep="")
proc.read_until_last_line_matches(os.path.basename(good_test.path))
proc.sendline("\x03")
proc.read_until_last_line_matches("TIME : %d s")
wait.wait_for(lambda: not proc.is_alive(), timeout=1)
# Make sure the good test will be really gone from the process table
def wait_until_no_goodtest():
good_test_processes = []
for p in psutil.pids():
p_obj = None
try:
p_obj = psutil.Process(p)
except psutil.NoSuchProcess:
pass
if p_obj is not None:
if good_test.path in " ".join(psutil.Process(p).cmdline()):
good_test_processes.append(p_obj)
return len(good_test_processes) == 0
wait.wait_for(wait_until_no_goodtest, timeout=2)
# Make sure the Killing test subprocess message is not there
self.assertNotIn("Killing test subprocess", proc.get_output())
# Make sure the Interrupted requested sentence is there
self.assertIn(
"Interrupt requested. Waiting 2 seconds for test to " "finish (ignoring new Ctrl+C until then)",
proc.get_output(),
)
def test_well_behaved_sigint(self):
"""
Make sure avocado can cleanly get out of a loop of well behaved tests.
"""
good_test_basename = ('goodtest-%s.py' %
data_factory.generate_random_string(5))
good_test = script.TemporaryScript(good_test_basename, GOOD_TEST,
'avocado_interrupt_test',
mode=DEFAULT_MODE)
good_test.save()
self.test_module = good_test.path
os.chdir(basedir)
cmd = ('%s run %s --sysinfo=off --job-results-dir %s ' %
(AVOCADO, self.test_module, self.tmpdir))
self.proc = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if not wait.wait_for(self._has_children, timeout=10):
process.kill_process_tree(self.proc.pid)
self.fail('Avocado did not start the test process.')
# This test will not ignore SIGINT, so it should
# terminate right away.
os.kill(self.proc.pid, signal.SIGINT)
if not wait.wait_for(self._is_finished, timeout=10):
process.kill_process_tree(self.proc.pid)
self.fail('Avocado was still running after receiving SIGINT '
'twice.')
self.assertTrue(wait.wait_for(self._no_test_in_process_table,
timeout=10), 'Avocado left processes behind.')
output = self.proc.stdout.read()
# Make sure the Interrupted requested sentence is there
self.assertIn(b'Interrupt requested. Waiting 2 seconds for test to '
b'finish (ignoring new Ctrl+C until then)', output)
# Make sure the Killing test subprocess message is not there
self.assertNotIn(b'Killing test subprocess', output)
def disrupt_abort_repair(self):
"""
Start repair target_node in background, then try to abort the repair streaming.
"""
self._set_current_disruption('AbortRepairMonkey')
self.log.debug("Start repair target_node in background")
thread1 = threading.Thread(target=self.repair_nodetool_repair)
thread1.start()
def repair_streaming_exists():
result = self.target_node.remoter.run('curl -X GET --header "Content-Type: application/json" --header "Accept: application/json" "http://127.0.0.1:10000/stream_manager/stream_manager/"')
return 'repair-' in result.stdout
wait.wait_for(func=repair_streaming_exists,
timeout=10,
step=0.01,
text='Wait for repair starts')
self.log.debug("Abort repair streaming by storage_service/force_terminate_repair API")
self.target_node.remoter.run('curl -X GET --header "Content-Type: application/json" --header "Accept: application/json" "http://127.0.0.1:10000/storage_service/force_terminate_repair"')
thread1.join(timeout=120)
self.log.debug("Execute a complete repair for target node")
self.repair_nodetool_repair()
def test_well_behaved_sigterm(self):
"""
Make sure avocado can cleanly get out of a loop of well behaved tests.
"""
good_test_basename = ('goodtest-%s.py' %
data_factory.generate_random_string(5))
good_test = script.TemporaryScript(good_test_basename, GOOD_TEST,
'avocado_interrupt_test',
mode=DEFAULT_MODE)
good_test.save()
self.test_module = good_test.path
os.chdir(BASEDIR)
cmd = ('%s run %s --sysinfo=off --job-results-dir %s ' %
(AVOCADO, self.test_module, self.tmpdir))
self.proc = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if not wait.wait_for(self._has_children, timeout=10):
process.kill_process_tree(self.proc.pid)
self.fail('Avocado did not start the test process.')
# This test should be terminated when the main process
# receives a SIGTERM.
os.kill(self.proc.pid, signal.SIGTERM)
if not wait.wait_for(self._is_finished, timeout=10):
process.kill_process_tree(self.proc.pid)
self.fail('Avocado was still running after receiving SIGINT '
'twice.')
self.assertTrue(wait.wait_for(self._no_test_in_process_table,
timeout=10), 'Avocado left processes behind.')
# Make sure the Interrupted test sentence is there
self.assertIn(b'Terminated\n', self.proc.stdout.read())
def wait_for_stop(self, timeout=60, step=0.1):
"""
Wait for libvirtd to stop.
:param timeout: Max wait time
:param step: Checking interval
"""
logging.debug('Waiting for libvirtd to stop')
if self.gdb:
return self.gdb.wait_for_stop(timeout=timeout)
else:
return wait.wait_for(
lambda: not self.running,
timeout=timeout,
step=step,
)
def test(self):
detected_distro = distro.detect()
parser = ConfigParser.ConfigParser()
config_file = self.datadir + '/services.cfg'
parser.read(config_file)
services_list = parser.get(detected_distro.name, 'services').split(',')
if 'PowerNV' in open('/proc/cpuinfo', 'r').read():
services_list.extend(['opal_errd', 'opal-prd'])
else:
services_list.extend(['rtas_errd'])
services_failed = []
runner = process.run
for service in services_list:
service_obj = SpecificServiceManager(service, runner)
self.log.info("Checking %s service" % service)
if service_obj.is_enabled() is False:
self.log.info("%s service Not Found !!!" % service)
services_failed.append(service)
continue
original_status = service_obj.status()
if original_status is True:
service_obj.stop()
if not wait_for(lambda: not service_obj.status(), 10):
self.log.info("Fail to stop %s service" % service)
services_failed.append(service)
continue
service_obj.start()
wait_for(service_obj.status, 10)
else:
service_obj.start()
if not wait_for(service_obj.status, 10):
self.log.info("Fail to start %s service" % service)
services_failed.append(service)
continue
service_obj.stop()
wait_for(lambda: not service_obj.status(), 10)
if not service_obj.status() is original_status:
self.log.info("Fail to restore original status of the %s"
"service" % service)
services_failed.append(service)
if services_failed:
self.fail("List of services failed: %s" % services_failed)
else:
self.log.info("All Services Passed the ON/OFF test")
def run_background_test(self):
args = (self.test, self.params, self.env, self.params["sub_type"])
bg_test = BackgroundTest(run_virt_sub_test, args)
bg_test.start()
wait_for(bg_test.is_alive, first=10, step=3, timeout=100)
return bg_test
def run_suite(self, params_list):
"""
Run one or more tests and report with test result.
:param params_list: a list of param dicts.
:return: a list of test failures.
"""
failures = []
self.sysinfo.start_job_hook()
self.result.start_tests()
q = queues.SimpleQueue()
test_suite = self.job.test_loader.discover(params_list, q)
for test_factory in test_suite:
p = multiprocessing.Process(target=self.run_test,
args=(test_factory, q,))
cycle_timeout = 1
time_started = time.time()
test_state = None
p.start()
early_state = q.get()
# At this point, the test is already initialized and we know
# for sure if there's a timeout set.
if 'timeout' in early_state['params'].keys():
timeout = float(early_state['params']['timeout'])
else:
timeout = self.DEFAULT_TIMEOUT
time_deadline = time_started + timeout
ctrl_c_count = 0
ignore_window = 2.0
ignore_time_started = time.time()
stage_1_msg_displayed = False
stage_2_msg_displayed = False
while True:
try:
if time.time() >= time_deadline:
logging.error("timeout")
os.kill(p.pid, signal.SIGUSR1)
break
wait.wait_for(lambda: not q.empty() or not p.is_alive(),
cycle_timeout, step=0.1)
if not q.empty():
test_state = q.get()
if not test_state['running']:
break
else:
self.job.result_proxy.notify_progress(True)
if test_state['paused']:
msg = test_state['paused_msg']
if msg:
self.job.view.notify(event='partial', msg=msg)
elif p.is_alive():
if ctrl_c_count == 0:
self.job.result_proxy.notify_progress()
else:
break
except KeyboardInterrupt:
time_elapsed = time.time() - ignore_time_started
ctrl_c_count += 1
if ctrl_c_count == 2:
if not stage_1_msg_displayed:
k_msg_1 = ("SIGINT sent to tests, waiting for their "
"reaction")
k_msg_2 = ("Ignoring Ctrl+C during the next "
"%d seconds so they can try to finish" %
ignore_window)
k_msg_3 = ("A new Ctrl+C sent after that will send a "
"SIGKILL to them")
self.job.view.notify(event='message', msg=k_msg_1)
self.job.view.notify(event='message', msg=k_msg_2)
self.job.view.notify(event='message', msg=k_msg_3)
stage_1_msg_displayed = True
ignore_time_started = time.time()
if (ctrl_c_count > 2) and (time_elapsed > ignore_window):
if not stage_2_msg_displayed:
k_msg_3 = ("Ctrl+C received after the ignore window. "
"Killing all active tests")
self.job.view.notify(event='message', msg=k_msg_3)
stage_2_msg_displayed = True
os.kill(p.pid, signal.SIGKILL)
# If test_state is None, the test was aborted before it ended.
if test_state is None:
if p.is_alive() and wait.wait_for(lambda: not q.empty(),
cycle_timeout, step=0.1):
test_state = q.get()
else:
early_state['time_elapsed'] = time.time() - time_started
test_state = self._fill_aborted_test_state(early_state)
test_log = logging.getLogger('avocado.test')
test_log.error('ERROR %s -> TestAbortedError: '
'Test aborted unexpectedly',
test_state['name'])
# don't process other tests from the list
if ctrl_c_count > 0:
self.job.view.notify(event='minor', msg='')
break
self.result.check_test(test_state)
if not status.mapping[test_state['status']]:
failures.append(test_state['name'])
runtime.CURRENT_TEST = None
self.result.end_tests()
self.sysinfo.end_job_hook()
return failures
def run_test(self, test_factory, queue, failures, job_deadline=0):
"""
Run a test instance inside a subprocess.
:param test_factory: Test factory (test class and parameters).
:type test_factory: tuple of :class:`avocado.test.Test` and dict.
:param queue: Multiprocess queue.
:type queue: :class`multiprocessing.Queue` instance.
:param failures: Store tests failed.
:type failures: list.
:param job_deadline: Maximum time to execute.
:type job_deadline: int.
"""
proc = multiprocessing.Process(target=self._run_test,
args=(test_factory, queue,))
cycle_timeout = 1
time_started = time.time()
test_state = None
proc.start()
early_state = queue.get()
if 'load_exception' in early_state:
self.job.view.notify(event='error',
msg='Avocado crashed during test load. '
'Some reports might have not been '
'generated. Aborting...')
sys.exit(exit_codes.AVOCADO_FAIL)
# At this point, the test is already initialized and we know
# for sure if there's a timeout set.
timeout = (early_state.get('params', {}).get('timeout') or
self.DEFAULT_TIMEOUT)
test_deadline = time_started + timeout
if job_deadline > 0:
deadline = min(test_deadline, job_deadline)
else:
deadline = test_deadline
ctrl_c_count = 0
ignore_window = 2.0
ignore_time_started = time.time()
stage_1_msg_displayed = False
stage_2_msg_displayed = False
first = 0.01
step = 0.1
while True:
try:
if time.time() >= deadline:
os.kill(proc.pid, signal.SIGUSR1)
break
wait.wait_for(lambda: not queue.empty() or not proc.is_alive(),
cycle_timeout, first, step)
if not queue.empty():
test_state = queue.get()
if not test_state['running']:
break
else:
self.job.result_proxy.notify_progress(True)
if test_state['paused']:
msg = test_state['paused_msg']
if msg:
self.job.view.notify(event='partial', msg=msg)
elif proc.is_alive():
if ctrl_c_count == 0:
self.job.result_proxy.notify_progress()
else:
break
except KeyboardInterrupt:
time_elapsed = time.time() - ignore_time_started
ctrl_c_count += 1
if ctrl_c_count == 2:
if not stage_1_msg_displayed:
k_msg_1 = ("SIGINT sent to tests, waiting for their "
"reaction")
k_msg_2 = ("Ignoring Ctrl+C during the next "
"%d seconds so they can try to finish" %
ignore_window)
k_msg_3 = ("A new Ctrl+C sent after that will send a "
"SIGKILL to them")
self.job.view.notify(event='message', msg=k_msg_1)
self.job.view.notify(event='message', msg=k_msg_2)
self.job.view.notify(event='message', msg=k_msg_3)
stage_1_msg_displayed = True
ignore_time_started = time.time()
if (ctrl_c_count > 2) and (time_elapsed > ignore_window):
if not stage_2_msg_displayed:
k_msg_3 = ("Ctrl+C received after the ignore window. "
"Killing all active tests")
self.job.view.notify(event='message', msg=k_msg_3)
stage_2_msg_displayed = True
os.kill(proc.pid, signal.SIGKILL)
# If test_state is None, the test was aborted before it ended.
if test_state is None:
if proc.is_alive() and wait.wait_for(lambda: not queue.empty(),
cycle_timeout, first, step):
test_state = queue.get()
else:
early_state['time_elapsed'] = time.time() - time_started
test_state = self._fill_aborted_test_state(early_state)
test_log = logging.getLogger('avocado.test')
test_log.error('ERROR %s -> TestAbortedError: '
'Test aborted unexpectedly',
test_state['name'])
# don't process other tests from the list
if ctrl_c_count > 0:
self.job.view.notify(event='minor', msg='')
self.result.check_test(test_state)
if not status.mapping[test_state['status']]:
failures.append(test_state['name'])
if ctrl_c_count > 0:
return False
return True
def test(self):
"""
Tests Multipath.
"""
msg = ""
multipath.form_conf_mpath_file()
plcy = "path_selector \"%s 0\"" % self.policy
multipath.form_conf_mpath_file(defaults_extra=plcy)
for path_dic in self.mpath_list:
self.log.debug("operating on paths", path_dic["paths"])
# Path Selector policy
self.log.info("changing Selector policy")
for policy in self.policies:
cmd = "path_selector \"%s 0\"" % policy
multipath.form_conf_mpath_file(defaults_extra=cmd)
if multipath.get_policy(path_dic["wwid"]) != policy:
msg += "%s for %s fails\n" % (policy, path_dic["wwid"])
# mutipath -f mpathX
if not multipath.flush_path(path_dic["name"]):
msg += "Flush of %s fails\n" % path_dic["name"]
self.mpath_svc.restart()
wait.wait_for(self.mpath_svc.status, timeout=10)
# Blacklisting wwid
self.log.info("Black listing WWIDs")
cmd = "wwid %s" % path_dic["wwid"]
multipath.form_conf_mpath_file(blacklist=cmd, defaults_extra=plcy)
if multipath.device_exists(path_dic["wwid"]):
msg += "Blacklist of %s fails\n" % path_dic["wwid"]
else:
multipath.form_conf_mpath_file(defaults_extra=plcy)
if not multipath.device_exists(path_dic["wwid"]):
msg += "Recovery of %s fails\n" % path_dic["wwid"]
# Blacklisting sdX
self.log.info("Black listing individual paths")
for disk in path_dic["paths"]:
cmd = "devnode %s" % disk
multipath.form_conf_mpath_file(blacklist=cmd,
defaults_extra=plcy)
if disk in multipath.get_paths(path_dic["wwid"]):
msg += "Blacklist of %s fails\n" % disk
multipath.form_conf_mpath_file(defaults_extra=plcy)
# Failing and reinstating individual paths eg: sdX
self.log.info(" Failing and reinstating the individual paths")
for path in path_dic["paths"]:
if multipath.fail_path(path) is False:
msg += "test failed while failing %s\n" % path
elif multipath.reinstate_path(path) is False:
msg += "test failed while reinstating %s\n" % path
self.mpath_svc.restart()
wait.wait_for(self.mpath_svc.status, timeout=10)
# Failing n-1 paths for short time and reinstating back
self.log.info("Failing and reinstating the n-1 paths")
for path in path_dic['paths'][:-1]:
if multipath.fail_path(path) is False:
msg += "%s did not failed in n-1 path fail\n" % path
time.sleep(180)
for path in path_dic['paths'][:-1]:
if multipath.reinstate_path(path) is False:
msg += "%s failed to recover in n-1 paths fails\n" % path
self.mpath_svc.restart()
wait.wait_for(self.mpath_svc.status, timeout=10)
# Failing all paths for short time and reinstating back
self.log.info("Failing and reinstating the All paths")
for path in path_dic['paths']:
if multipath.fail_path(path) is False:
msg += "%s did not failed in all paths fail\n" % path
time.sleep(180)
for path in path_dic['paths']:
if multipath.reinstate_path(path) is False:
msg += "%s did not recovered in all path fail\n" % path
self.mpath_svc.restart()
wait.wait_for(self.mpath_svc.status, timeout=10)
# Print errors
if msg:
self.fail("Some tests failed. Find details below:\n%s" % msg)
def test(self):
detected_distro = distro.detect()
parser = ConfigParser.ConfigParser()
parser.read(self.get_data('services.cfg'))
services_list = parser.get(detected_distro.name, 'services').split(',')
smm = SoftwareManager()
deps = []
if detected_distro.name == 'SuSE':
deps.extend(['ppc64-diag', 'libvirt-daemon'])
if detected_distro.version >= 15:
services_list.append('firewalld')
else:
services_list.append('SuSEfirewall2')
elif detected_distro.name == 'Ubuntu':
deps.extend(['opal-prd'])
if detected_distro.version >= 17:
services_list.remove('networking')
for package in deps:
if not smm.check_installed(package) and not smm.install(package):
self.cancel(' %s is needed for the test to be run' % package)
if 'PowerNV' in open('/proc/cpuinfo', 'r').read():
services_list.extend(['opal_errd', 'opal-prd'])
if os.path.exists('/proc/device-tree/bmc'):
services_list.remove('opal_errd')
else:
services_list.extend(['rtas_errd'])
services_failed = []
runner = process.run
for service in services_list:
service_obj = SpecificServiceManager(service, runner)
self.log.info("Checking %s service" % service)
if service_obj.is_enabled() is False:
self.log.info("%s service Not Found !!!" % service)
services_failed.append(service)
continue
original_status = service_obj.status()
if original_status is True:
service_obj.stop()
if not wait_for(lambda: not service_obj.status(), 10):
self.log.info("Fail to stop %s service" % service)
services_failed.append(service)
continue
service_obj.start()
wait_for(service_obj.status, 10)
else:
service_obj.start()
if not wait_for(service_obj.status, 10):
self.log.info("Fail to start %s service" % service)
services_failed.append(service)
continue
service_obj.stop()
wait_for(lambda: not service_obj.status(), 10)
if not service_obj.status() is original_status:
self.log.info("Fail to restore original status of the %s"
"service" % service)
services_failed.append(service)
if services_failed:
self.fail("List of services failed: %s" % services_failed)
else:
self.log.info("All Services Passed the ON/OFF test")
def setUp(self):
"""
Set up.
"""
self.policy = self.params.get('policy', default='service-time')
self.policies = ["service-time", "round-robin", "queue-length"]
# We will remove and add the policy back, so that this becomes
# the last member of the list. This is done so that in the
# policy change test later, this policy is set in the last
# iteration.
self.policies.remove(self.policy)
self.policies.append(self.policy)
# Install needed packages
dist = distro.detect()
pkg_name = ""
svc_name = ""
if dist.name == 'Ubuntu':
pkg_name += "multipath-tools"
svc_name = "multipath-tools"
elif dist.name == 'SuSE':
pkg_name += "multipath-tools"
svc_name = "multipathd"
else:
pkg_name += "device-mapper-multipath"
svc_name = "multipathd"
smm = SoftwareManager()
if not smm.check_installed(pkg_name) and not smm.install(pkg_name):
self.cancel("Can not install %s" % pkg_name)
# Check if given multipath devices are present in system
self.wwids = self.params.get('wwids', default='').split(',')
system_wwids = multipath.get_multipath_wwids()
wwids_to_remove = []
for wwid in self.wwids:
if wwid not in system_wwids:
self.log.info("%s not present in the system" % wwid)
wwids_to_remove.append(wwid)
for wwid in wwids_to_remove:
self.wwids.remove(wwid)
if self.wwids == []:
self.cancel("No Multipath Devices Given")
# Create service object
self.mpath_svc = service.SpecificServiceManager(svc_name)
self.mpath_svc.restart()
wait.wait_for(self.mpath_svc.status, timeout=10)
# Take a backup of current config file
self.mpath_file = "/etc/multipath.conf"
if os.path.isfile(self.mpath_file):
shutil.copyfile(self.mpath_file, "%s.bkp" % self.mpath_file)
self.mpath_list = []
# Find all details of multipath devices
for wwid in self.wwids:
if wwid not in process.system_output('multipath -ll',
ignore_status=True,
shell=True):
continue
self.mpath_dic = {}
self.mpath_dic["wwid"] = wwid
self.mpath_dic["name"] = multipath.get_mpath_name(wwid)
self.mpath_dic["paths"] = multipath.get_paths(wwid)
self.mpath_dic["policy"] = multipath.get_policy(wwid)
self.mpath_dic["size"] = multipath.get_size(wwid)
self.mpath_list.append(self.mpath_dic)
pprint(self.mpath_list)
