def __init__(
self,
loader_set,
stress_cmd,
timeout,
stress_num=1,
keyspace_num=1,
keyspace_name='', # pylint: disable=too-many-arguments
profile=None,
node_list=None,
round_robin=False,
client_encrypt=False,
stop_test_on_failure=True):
if not node_list:
node_list = []
self.loader_set = loader_set
self.stress_cmd = stress_cmd
self.timeout = timeout
self.stress_num = stress_num
self.keyspace_num = keyspace_num
self.keyspace_name = keyspace_name
self.profile = profile
self.node_list = node_list
self.round_robin = round_robin
self.client_encrypt = client_encrypt
self.stop_test_on_failure = stop_test_on_failure
self.executor = None
self.results_futures = []
self.shell_marker = generate_random_string(20)
# This marker is used to mark shell commands, in order to be able to kill them later
self.max_workers = 0
def __init__(self,
stress_cmd,
loader_set,
timeout,
node_list=None,
round_robin=False,
use_single_loader=False,
stop_test_on_failure=False,
stress_num=1,
credentials=None):
if not node_list:
node_list = []
self.loader_set = loader_set
self.stress_cmd = stress_cmd
self.timeout = timeout
self.use_single_loader = use_single_loader
self.round_robin = round_robin
self.node_list = node_list
self.stress_num = stress_num
if credentials and 'username=' not in self.stress_cmd:
self.stress_cmd += " -username {} -password {}".format(
*credentials)
self.stop_test_on_failure = stop_test_on_failure
self.executor = None
self.results_futures = []
self.shell_marker = generate_random_string(20)
self.max_workers = 0
def __init__(
self,
loader_set,
stress_cmd,
timeout,
stress_num=1,
node_list=None, # pylint: disable=too-many-arguments
round_robin=False,
params=None,
stop_test_on_failure=True):
self.loader_set: BaseLoaderSet = loader_set
self.stress_cmd = stress_cmd
self.timeout = timeout
self.stress_num = stress_num
self.node_list = node_list if node_list else []
self.round_robin = round_robin
self.params = params if params else dict()
self.loaders = []
self.executor = None
self.results_futures = []
self.max_workers = 0
self.shell_marker = generate_random_string(20)
self.shutdown_timeout = 180 # extra 3 minutes
self.stop_test_on_failure = stop_test_on_failure
def __init__(self, stress_cmd, loader_set, timeout, node_list=None, round_robin=False, use_single_loader=False,
stop_test_on_failure=False, stress_num=1, credentials=None):
if not node_list:
node_list = []
self.loader_set = loader_set
self.stress_cmd = stress_cmd
self.timeout = timeout
self.use_single_loader = use_single_loader
self.round_robin = round_robin
self.node_list = node_list
self.stress_num = stress_num
if credentials and 'username='******' -error-at-row-limit 1000' # make it fail after having 1000 errors at row
self.stop_test_on_failure = stop_test_on_failure
self.executor = None
self.results_futures = []
self.shell_marker = generate_random_string(20)
self.max_workers = 0
# Find stress mode:
# "scylla-bench -workload=sequential -mode=write -replication-factor=3 -partition-count=100"
# "scylla-bench -workload=uniform -mode=read -replication-factor=3 -partition-count=100"
self.sb_mode: ScyllaBenchModes = ScyllaBenchModes(re.search(r"-mode=(.+?) ", stress_cmd).group(1))
self.sb_workload: ScyllaBenchWorkloads = ScyllaBenchWorkloads(
re.search(r"-workload=(.+?) ", stress_cmd).group(1))
def __init__(self, *args):
self.logdir = os.path.join('/tmp', generate_random_string(10))
os.mkdir(self.logdir)
configure_logging(formatters={
'default': {
'()': MultilineMessagesFormatter,
'format': '%(message)s'
}
},
variables={'log_dir': self.logdir})
super().__init__(*args)
def __init__(self, loader_node, loader_idx, ycsb_log_filename):
super().__init__()
self.loader_node = loader_node
self.loader_idx = loader_idx
self.ycsb_log_filename = ycsb_log_filename
self.uuid = generate_random_string(10)
for operation in self.collectible_ops:
gauge_name = self.gauge_name(operation)
if gauge_name not in self.METRICS:
metrics = nemesis_metrics_obj()
self.METRICS[gauge_name] = metrics.create_gauge(
gauge_name, 'Gauge for ycsb metrics',
['instance', 'loader_idx', 'uuid', 'type'])
class TestProfileFactory(unittest.TestCase):
""" Test to illustrate profile factory usage """
tmpdir = os.path.join('/tmp', generate_random_string(10))
def __init__(self, *args, **kwargs):
self._subroutines = []
self.pr_factory = None
super().__init__(*args, **kwargs)
def tearDown(self) -> None:
if not self.pr_factory:
return
self.pr_factory.stop_and_dump()
dirs_to_expect = [
'PrometheusAlertManagerListener',
'main',
]
for sub in self._subroutines:
dirs_to_expect.append(sub._name) # pylint: disable=protected-access
not_found = []
for directory in dirs_to_expect:
bin_path = os.path.join(self.tmpdir, directory, 'stats.bin')
txt_path = os.path.join(self.tmpdir, directory, 'stats.txt')
for path in (bin_path, txt_path):
if not os.path.exists(path):
not_found.append(path)
bin_path = os.path.join(self.tmpdir, 'stats.bin')
txt_path = os.path.join(self.tmpdir, 'stats.txt')
for path in (bin_path, txt_path):
if not os.path.exists(path):
not_found.append(path)
self._subroutines = []
if self.pr_factory:
self.pr_factory.deactivate()
assert not not_found, "Following files were not found:\n" + '\n'.join(
not_found)
if os.path.exists(self.tmpdir):
shutil.rmtree(self.tmpdir)
def test_profile_enabled(self):
self.pr_factory = ProfilerFactory(self.tmpdir)
self.pr_factory.activate(autodump=False)
self._add_tests()
for sub in self._subroutines:
sub.start()
tmp = PrometheusAlertManagerListener('127.0.0.1')
tmp.start()
function_sleep()
function_while()
done = False
while not done:
done = True
for sub in self._subroutines:
if sub.is_alive():
done = False
break
tmp.stop()
def test_profile_disabled(self):
self._add_tests()
for sub in self._subroutines:
sub.start()
tmp = PrometheusAlertManagerListener('127.0.0.1')
tmp.start()
function_sleep()
function_while()
done = False
while not done:
done = True
for sub in self._subroutines:
if sub.is_alive():
done = False
break
tmp.stop()
def _add_tests(self): # pylint: disable=too-many-statements
self._subroutines.append(
Thread(target=thread_body, name="Thread.daemon", daemon=True))
self._subroutines.append(
LibThread(target=thread_body,
name="lib.Thread.daemon",
daemon=True))
self._subroutines.append(
ThreadCustomClass(target=thread_body,
name="Thread.CustomClass.daemon",
daemon=True))
self._subroutines.append(
LibThreadCustomClass(target=thread_body,
name="lib.Thread.CustomClass.daemon",
daemon=True))
self._subroutines.append(
ThreadCustomClassWithRun(target=thread_body,
name="Thread.CustomClassWithRun.daemon",
daemon=True))
self._subroutines.append(
LibThreadCustomClassWithRun(
target=thread_body,
name="lib.Thread.CustomClassWithRun.daemon",
daemon=True))
self._subroutines.append(
threading.Thread(target=thread_body,
name='threading.Thread.daemon',
daemon=True))
self._subroutines.append(
ThreadingThreadCustomClass(
target=thread_body,
name="threading.Thread.CustomClass.daemon",
daemon=True))
self._subroutines.append(
LibThreadingThreadCustomClass(
target=thread_body,
name="lib.threading.Thread.CustomClass.daemon",
daemon=True))
self._subroutines.append(
ThreadingThreadCustomClassWithRun(
name="threading.Thread.CustomClassWithRun.daemon",
daemon=True))
self._subroutines.append(
LibThreadingThreadCustomClassWithRun(
name="lib.threading.Thread.CustomClassWithRun.daemon",
daemon=True))
self._subroutines.append(
pt(target=thread_body,
name='ProfilerableThread.daemon',
daemon=True))
self._subroutines.append(
LibProfileableThread(target=thread_body,
name='lib.ProfilerableThread.daemon',
daemon=True))
self._subroutines.append(
ProfileableThreadCustomClass(
target=thread_body,
name="ProfilerableThread.CustomClass.daemon",
daemon=True))
self._subroutines.append(
LibProfileableThreadCustomClass(
target=thread_body,
name="lib.ProfilerableThread.CustomClass.daemon",
daemon=True))
self._subroutines.append(
ProfileableThreadCustomClassWithRun(
name="ProfilerableThread.CustomClassWithRun.daemon",
daemon=True))
self._subroutines.append(
LibProfileableThreadCustomClassWithRun(
name="lib.ProfilerableThread.CustomClassWithRun.daemon",
daemon=True))
self._subroutines.append(
Thread(target=thread_body, name="Thread", daemon=False))
self._subroutines.append(
LibThread(target=thread_body, name="lib.Thread", daemon=False))
self._subroutines.append(
ThreadCustomClass(target=thread_body,
name="Thread.CustomClass",
daemon=False))
self._subroutines.append(
LibThreadCustomClass(target=thread_body,
name="lib.Thread.CustomClass",
daemon=False))
self._subroutines.append(
ThreadCustomClassWithRun(target=thread_body,
name="Thread.CustomClassWithRun",
daemon=False))
self._subroutines.append(
LibThreadCustomClassWithRun(target=thread_body,
name="lib.Thread.CustomClassWithRun",
daemon=False))
self._subroutines.append(
threading.Thread(target=thread_body,
name='threading.Thread',
daemon=False))
self._subroutines.append(
LibThreadingThreadCustomClass(
target=thread_body,
name="lib.ThreadingThread.CustomClass",
daemon=False))
self._subroutines.append(
LibThreadingThreadCustomClassWithRun(
name="lib.ThreadingThread.CustomClassWithRun", daemon=False))
self._subroutines.append(
pt(target=thread_body,
name='ProfilerableThread.CustomClass',
daemon=False))
self._subroutines.append(
LibProfileableThread(target=thread_body,
name='lib.ProfilerableThread.CustomClass',
daemon=False))
self._subroutines.append(
ProfileableThreadCustomClass(target=thread_body,
name="ProfilerableThread.CustomClass",
daemon=False))
self._subroutines.append(
LibProfileableThreadCustomClass(
target=thread_body,
name="lib.ProfilerableThread.CustomClass",
daemon=False))
self._subroutines.append(
ProfileableThreadCustomClassWithRun(
name="ProfilerableThread.CustomClassWithRun", daemon=False))
self._subroutines.append(
LibProfileableThreadCustomClassWithRun(
name="lib.ProfilerableThread.CustomClassWithRun",
daemon=False))
self._subroutines.append(
Process(target=process_body, name="Process.daemon", daemon=True))
self._subroutines.append(
LibProcess(target=process_body,
name="lib.Process.daemon",
daemon=True))
self._subroutines.append(
ProcessCustomClass(target=process_body,
name="Process.CustomClass.daemon",
daemon=True))
self._subroutines.append(
LibProcessCustomClass(target=process_body,
name="lib.Process.CustomClass.daemon",
daemon=True))
self._subroutines.append(
ProcessCustomClassWithRun(target=process_body,
name="Process.CustomClassWithRun.daemon",
daemon=True))
self._subroutines.append(
LibProcessCustomClassWithRun(
target=process_body,
name="lib.Process.CustomClassWithRun.daemon",
daemon=True))
self._subroutines.append(
multiprocessing.Process(target=process_body,
name='multiprocessing.Process.daemon',
daemon=True))
self._subroutines.append(
MultiprocessingProcessCustomClass(
target=process_body,
name="multiprocessing.Process.CustomClass.daemon",
daemon=True))
self._subroutines.append(
LibMultiprocessingProcessCustomClass(
target=process_body,
name="lib.multiprocessing.Process.CustomClass.daemon",
daemon=True))
self._subroutines.append(
MultiprocessingProcessCustomClassWithRun(
name="multiprocessing.Process.CustomClassWithRun.daemon",
daemon=True))
self._subroutines.append(
LibMultiprocessingProcessCustomClassWithRun(
name="lib.multiprocessing.Process.CustomClassWithRun.daemon",
daemon=True))
self._subroutines.append(
pp(target=thread_body,
name='ProfilerableProcess.CustomClass.daemon',
daemon=True))
self._subroutines.append(
LibProfileableProcess(
target=process_body,
name='lib.ProfilerableProcess.CustomClass.daemon',
daemon=True))
self._subroutines.append(
ProfileableProcessCustomClass(
target=process_body,
name="ProfilerableProcess.CustomClass.daemon",
daemon=True))
self._subroutines.append(
LibProfileableProcessCustomClass(
target=process_body,
name="lib.ProfilerableProcess.CustomClass.daemon",
daemon=True))
self._subroutines.append(
ProfileableProcessCustomClassWithRun(
name="ProfilerableProcess.CustomClassWithRun.daemon",
daemon=True))
self._subroutines.append(
LibProfileableProcessCustomClassWithRun(
name="lib.ProfilerableProcess.CustomClassWithRun.daemon",
daemon=True))
self._subroutines.append(
Process(target=thread_body, name="Process", daemon=False))
self._subroutines.append(
LibProcess(target=thread_body, name="lib.Process", daemon=False))
self._subroutines.append(
ProcessCustomClass(target=process_body,
name="Process.CustomClass",
daemon=False))
self._subroutines.append(
LibProcessCustomClass(target=process_body,
name="lib.Process.CustomClass",
daemon=False))
self._subroutines.append(
ProcessCustomClassWithRun(target=process_body,
name="Process.CustomClassWithRun",
daemon=False))
self._subroutines.append(
LibProcessCustomClassWithRun(target=process_body,
name="lib.Process.CustomClassWithRun",
daemon=False))
self._subroutines.append(
multiprocessing.Process(target=process_body,
name='multiprocessing.Process',
daemon=False))
self._subroutines.append(
MultiprocessingProcessCustomClass(
target=process_body,
name="MultiprocessingProcess.CustomClass",
daemon=False))
self._subroutines.append(
LibMultiprocessingProcessCustomClass(
target=process_body,
name="lib.MultiprocessingProcess.CustomClass",
daemon=False))
self._subroutines.append(
MultiprocessingProcessCustomClassWithRun(
name="MultiprocessingProcess.CustomClassWithRun",
daemon=False))
self._subroutines.append(
LibMultiprocessingProcessCustomClassWithRun(
name="lib.MultiprocessingProcess.CustomClassWithRun",
daemon=False))
self._subroutines.append(
pp(target=process_body,
name='ProfilerableProcess.CustomClass',
daemon=False))
self._subroutines.append(
LibProfileableProcess(target=process_body,
name='lib.ProfilerableProcess.CustomClass',
daemon=False))
self._subroutines.append(
ProfileableProcessCustomClass(
target=process_body,
name="ProfilerableProcess.CustomClass",
daemon=False))
self._subroutines.append(
LibProfileableProcessCustomClass(
target=process_body,
name="lib.ProfilerableProcess.CustomClass",
daemon=False))
self._subroutines.append(
ProfileableProcessCustomClassWithRun(
name="ProfilerableProcess.CustomClassWithRun", daemon=False))
self._subroutines.append(
LibProfileableProcessCustomClassWithRun(
name="lib.ProfilerableProcess.CustomClassWithRun",
daemon=False))
