def test_successful_parallel_run_func_returning_single_value(self):
parallel_object = ParallelObject(self.rand_timeouts,
timeout=30,
num_workers=len(self.rand_timeouts))
results = parallel_object.run(dummy_func_return_single)
returned_results = [r.result for r in results]
self.assertListEqual(returned_results, self.rand_timeouts)
def test_exception_raised_in_thread_by_func(self):
with self.assertRaises(DummyException):
parallel_object = ParallelObject(self.rand_timeouts,
timeout=30,
num_workers=len(
self.rand_timeouts))
parallel_object.run(dummy_func_raising_exception)
def test_successful_parallel_run_func_returning_tuple(self):
parallel_object = ParallelObject(self.rand_timeouts, timeout=self.max_timout + 2,
num_workers=len(self.rand_timeouts))
results = parallel_object.run(dummy_func_return_tuple)
returned_results = [r.result for r in results]
expected_results = [(timeout, 'test') for timeout in self.rand_timeouts]
self.assertListEqual(returned_results, expected_results)
def test_raised_exception_by_timeout(self):
with self.assertRaises(concurrent.futures.TimeoutError):
parallel_object = ParallelObject(self.rand_timeouts,
timeout=10,
num_workers=len(
self.rand_timeouts))
parallel_object.run(dummy_func_return_tuple)
def run_workload(self, stress_cmd, nemesis=False, sub_type=None):
def _run_workload(scylla_cluster_stats, stress_cmd, nemesis):
scylla_cluster_stats.run_workload(stress_cmd=stress_cmd, nemesis=nemesis)
self.log.info("Running workload in parallel with following command:\n%s", stress_cmd)
object_set = ParallelObject(timeout=self.load_iteration_timeout_sec, objects=[
[scs, stress_cmd, nemesis] for scs in self.scylla_clusters_stats])
object_set.run(func=_run_workload, unpack_objects=True, ignore_exceptions=False)
def test_raised_exception_by_timeout(self):
test_timeout = min(self.rand_timeouts)
start_time = time.time()
with self.assertRaises(concurrent.futures.TimeoutError):
parallel_object = ParallelObject(self.rand_timeouts, timeout=test_timeout)
parallel_object.run(dummy_func_return_tuple)
run_time = int(time.time() - start_time)
self.assertAlmostEqual(first=test_timeout, second=run_time, delta=1)
def test_successfull_parallel_run_func_accepted_list_as_parameter(self):
parallel_object = ParallelObject(self.list_as_arg,
timeout=30,
num_workers=len(self.list_as_arg))
results = parallel_object.run(dummy_func_accepts_list_as_parameter)
returned_results = [r.result for r in results]
expected_results = [r[0][1] for r in self.list_as_arg]
self.assertListEqual(returned_results, expected_results)
def run_mixed_workload(self, nemesis: bool = False):
def _run_mixed_workload(scylla_cluster_stats, nemesis):
scylla_cluster_stats.run_mixed_workload(nemesis=nemesis)
self.log.info("Running 'mixed' workload operation in parallel")
object_set = ParallelObject(timeout=self.load_iteration_timeout_sec, objects=[
[scs, nemesis] for scs in self.scylla_clusters_stats])
object_set.run(func=_run_mixed_workload, unpack_objects=True, ignore_exceptions=False)
def test_less_number_of_workers_than_length_of_iterable(self):
parallel_object = ParallelObject(self.rand_timeouts,
timeout=30,
num_workers=2)
results = parallel_object.run(dummy_func_return_tuple)
returned_results = [r.result for r in results]
expected_results = [(timeout, 'test')
for timeout in self.rand_timeouts]
self.assertListEqual(returned_results, expected_results)
def install_benchmark_tools(self):
try:
parallel = ParallelObject(self._benchmark_runners, timeout=300)
parallel.run(lambda x: x.install_benchmark_tools(),
ignore_exceptions=True)
except TimeoutError as exc:
LOGGER.warning(
"Ran into TimeoutError while installing benchmark tools: Exception:\n%s",
exc)
def test_unpack_args_for_func(self):
parallel_object = ParallelObject(self.unpacking_args,
timeout=30,
num_workers=2)
results = parallel_object.run(dummy_func_with_several_parameters)
returned_results = [r.result for r in results]
expected_results = [(timeout, msg)
for timeout, msg in self.unpacking_args] # pylint: disable=unnecessary-comprehension
self.assertListEqual(returned_results, expected_results)
def test_unpack_kwargs_for_func(self):
parallel_object = ParallelObject(self.unpacking_kwargs,
timeout=30,
num_workers=2)
results = parallel_object.run(dummy_func_with_several_parameters)
returned_results = [r.result for r in results]
expected_results = [(d["timeout"], d["msg"])
for d in self.unpacking_kwargs]
self.assertListEqual(returned_results, expected_results)
def run_benchmarks(self):
try:
parallel = ParallelObject(self._benchmark_runners, timeout=300)
parallel.run(lambda x: x.run_benchmarks(), ignore_exceptions=True)
except TimeoutError as exc:
LOGGER.warning(
"Run into TimeoutError during running benchmarks. Exception:\n%s",
exc)
self._collect_benchmark_output()
def test_ignore_exception_by_timeout(self):
parallel_object = ParallelObject(self.rand_timeouts, timeout=min(self.rand_timeouts))
results = parallel_object.run(dummy_func_return_tuple, ignore_exceptions=True)
for res_obj in results:
if res_obj.exc:
self.assertIsNone(res_obj.result)
self.assertIsInstance(res_obj.exc, concurrent.futures.TimeoutError)
else:
self.assertIsNone(res_obj.exc)
self.assertIn(res_obj.result, [(timeout, 'test') for timeout in self.rand_timeouts])
def test_ignore_exception_raised_in_func_and_get_results(self):
parallel_object = ParallelObject(self.rand_timeouts, timeout=self.max_timout + 2)
results = parallel_object.run(dummy_func_raising_exception, ignore_exceptions=True)
for res_obj in results:
self.assertIsNotNone(res_obj.obj)
if res_obj.exc:
self.assertIsNone(res_obj.result)
self.assertIsInstance(res_obj.exc, DummyException)
else:
self.assertIsNone(res_obj.exc)
self.assertEqual(res_obj.result, "done")
def test_ignore_exception_raised_in_func_and_get_results(self):
parallel_object = ParallelObject(self.rand_timeouts,
timeout=30,
num_workers=len(self.rand_timeouts))
results = parallel_object.run(dummy_func_raising_exception,
ignore_exceptions=True)
for res_obj in results:
if res_obj.exc:
self.assertIsNone(res_obj.result)
self.assertIsInstance(res_obj.exc, DummyException)
else:
self.assertIsNone(res_obj.exc)
self.assertListEqual(res_obj.result, "done")
def get_scylla_urls_from_repository(repo_details):
urls = set()
for url in repo_details.urls:
for (url_regex, url_format) in FILE_REGEX_DICT[repo_details.type]:
match = url_regex.match(url)
if not match:
# Continue until find the correct Regex
continue
full_url = url_format.format(**match.groupdict())
# for scylla-manager we never used a noarch key
basearch_list = ["x86_64"] if 'scylla-manager' in full_url else [
"x86_64", "noarch"
]
for basearch in basearch_list:
urls.add(
Template(full_url).substitute(basearch=basearch,
releasever='7'))
# We found the correct regex and we can continue to next URL
break
urls = list(urls)
ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(
func=lambda _url: get_url_content(url=_url, return_url_data=False))
return urls
def get_scylla_urls_from_repository(repo_details):
urls = set()
for url in repo_details.urls:
match = None
url_format = None
for (url_regex, url_format) in FILE_REGEX_DICT[repo_details.type]:
match = url_regex.match(url)
if match:
break
if not match or not url_format:
continue
url_details = {**match.groupdict()}
archs = url_details.get('arch', None)
if archs is None:
archs = [None]
else:
archs = archs.split(',')
for arch in archs:
if arch == '':
continue
full_url = url_format.format(**{**url_details, 'arch': arch})
# for scylla-manager we never used a noarch key
basearch_list = ["x86_64"] if 'scylla-manager' in full_url else ["x86_64", "noarch"]
for basearch in basearch_list:
urls.add(Template(full_url).substitute(basearch=basearch, releasever='7'))
# We found the correct regex and we can continue to next URL
ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=lambda _url: get_url_content(
url=_url, return_url_data=False))
return urls
def get_all_versions_from_debian_repository(urls: set[str]) -> set[str]:
def get_version(url: str) -> set[str]:
data = '\n'.join(get_url_content(url=url))
# Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
return set(major_versions)
threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
result = set.union(*[thread.result for thread in threads])
return result
def stop_reshape_compaction(self):
"""
Test that we can stop a reshape compaction with <nodetool stop RESHAPE>.
To trigger a reshape compaction, the current CompactionStrategy
must not be aligned with the shape of the stored sstables. In
this case we're setting the new strategy to
TimeWindowCompactionStrategy with a very small time window.
1. Flush the memtable data to sstables.
2. Copy sstable files to the upload and staging dirs.
3. Change the compaction strategy to TimeWindowCompactionStrategy.
4. Run <nodetool refresh> command to trigger the compaction.
5. Stop the compaction mid-flight with <nodetool stop RESHAPE>.
6. Grep the logs for a line informing of the reshape compaction
being stopped due to a user request.
7. Assert that we found the line we were grepping for.
"""
node = self.node
compaction_ops = CompactionOps(cluster=self.db_cluster, node=node)
timeout = 600
def _trigger_reshape(node: BaseNode, tester, keyspace: str = "keyspace1"):
twcs = {'class': 'TimeWindowCompactionStrategy', 'compaction_window_size': 1,
'compaction_window_unit': 'MINUTES', 'max_threshold': 1, 'min_threshold': 1}
compaction_ops.trigger_flush()
tester.wait_no_compactions_running()
LOGGER.info("Copying data files to ./staging and ./upload directories...")
keyspace_dir = f'/var/lib/scylla/data/{keyspace}'
cf_data_dir = node.remoter.run(f"ls {keyspace_dir}").stdout.splitlines()[0]
full_dir_path = f"{keyspace_dir}/{cf_data_dir}"
upload_dir = f"{full_dir_path}/upload"
staging_dir = f"{full_dir_path}/staging"
cp_cmd_upload = f"cp -p {full_dir_path}/m* {upload_dir}"
cp_cmd_staging = f"cp -p {full_dir_path}/m* {staging_dir}"
node.remoter.sudo(cp_cmd_staging)
node.remoter.sudo(cp_cmd_upload)
LOGGER.info("Finished copying data files to ./staging and ./upload directories.")
cmd = f"ALTER TABLE standard1 WITH compaction={twcs}"
node.run_cqlsh(cmd=cmd, keyspace="keyspace1")
node.run_nodetool("refresh -- keyspace1 standard1")
trigger_func = partial(_trigger_reshape,
node=node,
tester=self)
watch_func = partial(compaction_ops.stop_on_user_compaction_logged,
node=node,
mark=node.mark_log(),
watch_for="Reshape keyspace1.standard1",
timeout=timeout,
stop_func=compaction_ops.stop_reshape_compaction)
try:
ParallelObject(objects=[trigger_func, watch_func], timeout=timeout).call_objects()
finally:
self._grep_log_and_assert(node)
def get_branch_version_from_debian_repository(urls):
def get_version(url):
data = '\n'.join(get_url_content(url=url))
# Get only the major version (i.e. "2019.1.1-0.20190709.9f724fedb-1~stretch", get only "2019.1.1")
major_versions = [version.split('-', maxsplit=1)[0] for version in REPO_VERSIONS_REGEX.findall(data)]
if not major_versions:
return ""
return max(set(major_versions), key=major_versions.count)
threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
result = [thread.result for thread in threads]
return max(result, key=result.count)
def stop_upgrade_compaction(self):
"""
Test that we can stop an upgrade compaction with <nodetool stop UPGRADE>.
Prerequisite:
The initial setup in scylla.yaml must include 2 settings:
enable_sstables_mc_format: true
enable_sstables_md_format: false
This is necessary for the test to be able to go from the legacy
"mc" sstable format to the newer "md" format.
1. Flush the data from memtables to sstables.
2. Stop scylla on a given node.
3. Update the scylla.yaml configuration to enable the mc
sstable format.
4. Restart scylla.
5. Trigger the upgrade compaction using the API request.
6. Stop the compaction mid-flight with <nodetool stop UPGRADE>.
7. Grep the logs for a line informing of the upgrade compaction being
stopped due to a user request.
8. Assert that we found the line we were grepping for.
"""
compaction_ops = CompactionOps(cluster=self.db_cluster, node=self.node)
timeout = 300
upgraded_configuration_options = {
"enable_sstables_mc_format": False,
"enable_sstables_md_format": True
}
trigger_func = partial(compaction_ops.trigger_upgrade_compaction)
watch_func = partial(compaction_ops.stop_on_user_compaction_logged,
node=self.node,
mark=self.node.mark_log(),
watch_for="Upgrade keyspace1.standard1",
timeout=timeout,
stop_func=compaction_ops.stop_upgrade_compaction)
def _upgrade_sstables_format(node: BaseNode):
LOGGER.info("Upgrading sstables format...")
with node.remote_scylla_yaml() as scylla_yaml:
scylla_yaml.update(upgraded_configuration_options)
try:
compaction_ops.trigger_flush()
self.node.stop_scylla()
_upgrade_sstables_format(self.node)
self.node.start_scylla()
self.wait_no_compactions_running()
ParallelObject(objects=[trigger_func, watch_func],
timeout=timeout).call_objects()
self._grep_log_and_assert(self.node)
finally:
self.node.running_nemesis = False
def get_branch_version_from_centos_repository(urls):
def get_version(url):
data = '\n'.join(get_url_content(url=url))
primary_path = PRIMARY_XML_GZ_REGEX.search(data).groups()[0]
xml_url = url.replace(REPOMD_XML_PATH, primary_path)
parser = Parser(url=xml_url)
major_versions = [package['version'][1]['ver'] for package in parser.getList()]
return max(set(major_versions), key=major_versions.count)
threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
result = [thread.result for thread in threads]
return max(result, key=result.count)
def get_all_versions_from_centos_repository(urls: set[str]) -> set[str]:
def get_version(url: str) -> set[str]:
data = '\n'.join(get_url_content(url=url))
primary_path = PRIMARY_XML_GZ_REGEX.search(data).groups()[0]
xml_url = url.replace(REPOMD_XML_PATH, primary_path)
parser = Parser(url=xml_url)
major_versions = [package['version'][1]['ver'] for package in parser.getList()]
return set(major_versions)
threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_version)
result = set.union(*[thread.result for thread in threads])
return result
def preload_data(self):
def _preload_data(scylla_cluster_stats):
prepare_write_cmd = scylla_cluster_stats.params.get('prepare_write_cmd')
db_cluster_name = scylla_cluster_stats.db_cluster.name
if not prepare_write_cmd:
self.log.warning(
"No prepare command defined in YAML for the '%s' cluster",
db_cluster_name)
return
self.log.info("Running preload command for the '%s' cluster", db_cluster_name)
scylla_cluster_stats.create_test_stats(
sub_type='write-prepare', doc_id_with_timestamp=True)
stress_queue, params = [], {
'prefix': 'preload-',
}
if self.params.get('round_robin'):
self.log.debug(
"'%s' DB cluster: Populating data using round_robin", db_cluster_name)
params.update({'stress_num': 1, 'round_robin': True})
for stress_cmd in prepare_write_cmd:
params.update({'stress_cmd': stress_cmd})
# Run all stress commands
params.update(dict(stats_aggregate_cmds=False))
self.log.debug("'%s' DB cluster: RUNNING stress cmd: %s",
db_cluster_name, stress_cmd)
stress_queue.append(scylla_cluster_stats.run_stress_thread(**params))
for stress in stress_queue:
scylla_cluster_stats.get_stress_results(queue=stress, store_results=False)
scylla_cluster_stats.update_test_details()
self.log.info("Running preload operation in parallel on all the DB clusters")
object_set = ParallelObject(
timeout=self.load_iteration_timeout_sec,
objects=[[scs] for scs in self.scylla_clusters_stats],
num_workers=len(self.scylla_clusters_stats),
)
object_set.run(func=_preload_data, unpack_objects=True, ignore_exceptions=False)
def collect_logs(self, local_search_path=None):
def collect_logs_per_node(node):
LOGGER.info('Collecting logs on host: %s', node.name)
remote_node_dir = self.create_remote_storage_dir(node)
local_node_dir = os.path.join(self.local_dir, node.name)
for log_entity in self.log_entities:
try:
log_entity.collect(node,
local_node_dir,
remote_node_dir,
local_search_path=local_search_path)
except Exception as details: # pylint: disable=unused-variable, broad-except
LOGGER.error(
"Error occured during collecting on host: %s\n%s",
node.name, details)
LOGGER.debug("Nodes list %s", [node.name for node in self.nodes])
if not self.nodes:
LOGGER.warning(
f'No nodes found for {self.cluster_log_type} cluster. Logs will not be collected'
)
return None
try:
workers_number = int(len(self.nodes) / 2)
workers_number = len(
self.nodes) if workers_number < 2 else workers_number
ParallelObject(self.nodes,
num_workers=workers_number,
timeout=self.collect_timeout).run(
collect_logs_per_node, ignore_exceptions=True)
except Exception as details: # pylint: disable=broad-except
LOGGER.error('Error occured during collecting logs %s', details)
if not os.listdir(self.local_dir):
LOGGER.warning('Directory %s is empty', self.local_dir)
return None
final_archive = self.archive_dir_with_zip64(self.local_dir)
if not final_archive:
return None
s3_link = self.upload_logs(final_archive,
"{0.test_id}/{0.current_run}".format(self))
remove_files(self.local_dir)
remove_files(final_archive)
return s3_link
def test_unpack_args_for_func(self):
parallel_object = ParallelObject(self.unpacking_args, timeout=self.max_timout + 2, num_workers=2)
results = parallel_object.run(dummy_func_with_several_parameters, unpack_objects=True)
returned_results = [r.result for r in results]
expected_results = [tuple(item) for item in self.unpacking_args]
self.assertListEqual(returned_results, expected_results)
def test_successfull_parallel_run_func_accepted_list_as_parameter(self):
parallel_object = ParallelObject(self.list_as_arg, timeout=self.max_timout + 2)
results = parallel_object.run(dummy_func_accepts_list_as_parameter, unpack_objects=True)
returned_results = [r.result for r in results]
expected_results = [r[0][1] for r in self.list_as_arg]
self.assertListEqual(returned_results, expected_results)
def test_parallel_object_exception_raised(self):
with self.assertRaises(ParallelObjectException):
parallel_object = ParallelObject(self.rand_timeouts, timeout=self.max_timout + 2)
parallel_object.run(dummy_func_raising_exception)
def get_branch_version_for_multiple_repositories(urls):
threads = ParallelObject(objects=urls, timeout=SCYLLA_URL_RESPONSE_TIMEOUT).run(func=get_branch_version)
return [thread.result for thread in threads]