def get_results(self):
ret = []
results = []
LOGGER.debug('Wait for %s stress threads results', self.max_workers)
for future in concurrent.futures.as_completed(self.results_futures,
timeout=self.timeout):
results.append(future.result())
for _, result in results:
if not result:
# Silently skip if stress command threw error, since it was already reported in _run_stress
continue
output = result.stdout + result.stderr
try:
lines = output.splitlines()
node_cs_res = BaseLoaderSet._parse_cs_summary(lines) # pylint: disable=protected-access
if node_cs_res:
ret.append(node_cs_res)
except Exception as exc: # pylint: disable=broad-except
CassandraStressEvent.failure(
node="",
errors=[
f"Failed to process stress summary due to {exc}",
])
return ret
def _run_stress(self, node, loader_idx, cpu_idx, keyspace_idx): # pylint: disable=too-many-locals
stress_cmd = self.create_stress_cmd(node, loader_idx, keyspace_idx)
if self.profile:
with open(self.profile) as profile_file:
LOGGER.info('Profile content:\n%s', profile_file.read())
node.remoter.send_files(self.profile,
os.path.join(
'/tmp',
os.path.basename(self.profile)),
delete_dst=True)
stress_cmd_opt = stress_cmd.split()[1]
LOGGER.info('Stress command:\n%s', stress_cmd)
os.makedirs(node.logdir, exist_ok=True)
log_file_name = \
os.path.join(node.logdir, f'cassandra-stress-l{loader_idx}-c{cpu_idx}-k{keyspace_idx}-{uuid.uuid4()}.log')
LOGGER.debug('cassandra-stress local log: %s', log_file_name)
# This tag will be output in the header of c-stress result,
# we parse it to know the loader & cpu info in _parse_cs_summary().
tag = f'TAG: loader_idx:{loader_idx}-cpu_idx:{cpu_idx}-keyspace_idx:{keyspace_idx}'
if self.stress_num > 1:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; taskset -c {cpu_idx} {stress_cmd}'
else:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {stress_cmd}'
node_cmd = f'echo {tag}; {node_cmd}'
result = None
CassandraStressEvent.start(node=node, stress_cmd=stress_cmd).publish()
with CassandraStressExporter(instance_name=node.ip_address,
metrics=nemesis_metrics_obj(),
stress_operation=stress_cmd_opt,
stress_log_filename=log_file_name,
loader_idx=loader_idx, cpu_idx=cpu_idx), \
CassandraStressEventsPublisher(node=node, cs_log_filename=log_file_name):
try:
result = node.remoter.run(cmd=node_cmd,
timeout=self.timeout,
log_file=log_file_name)
except Exception as exc:
event_type = CassandraStressEvent.failure if self.stop_test_on_failure else CassandraStressEvent.error
event_type(node=node,
stress_cmd=stress_cmd,
log_file_name=log_file_name,
errors=[
format_stress_cmd_error(exc),
]).publish()
CassandraStressEvent.finish(node=node,
stress_cmd=stress_cmd,
log_file_name=log_file_name).publish()
return node, result
def _run_stress(self, node, loader_idx, cpu_idx, keyspace_idx): # pylint: disable=too-many-locals
stress_cmd = self.create_stress_cmd(node, loader_idx, keyspace_idx)
if self.profile:
with open(self.profile, encoding="utf-8") as profile_file:
LOGGER.info('Profile content:\n%s', profile_file.read())
node.remoter.send_files(self.profile,
os.path.join(
'/tmp',
os.path.basename(self.profile)),
delete_dst=True)
# Get next word after `cassandra-stress' in stress_cmd.
# Do it this way because stress_cmd can contain env variables before `cassandra-stress'.
stress_cmd_opt = stress_cmd.split("cassandra-stress",
1)[1].split(None, 1)[0]
LOGGER.info('Stress command:\n%s', stress_cmd)
os.makedirs(node.logdir, exist_ok=True)
log_file_name = \
os.path.join(node.logdir, f'cassandra-stress-l{loader_idx}-c{cpu_idx}-k{keyspace_idx}-{uuid.uuid4()}.log')
LOGGER.debug('cassandra-stress local log: %s', log_file_name)
# This tag will be output in the header of c-stress result,
# we parse it to know the loader & cpu info in _parse_cs_summary().
tag = f'TAG: loader_idx:{loader_idx}-cpu_idx:{cpu_idx}-keyspace_idx:{keyspace_idx}'
if self.stress_num > 1:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; taskset -c {cpu_idx} {stress_cmd}'
else:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {stress_cmd}'
node_cmd = f'echo {tag}; {node_cmd}'
result = None
# disable logging for cassandra stress
node.remoter.run("cp /etc/scylla/cassandra/logback-tools.xml .",
ignore_status=True)
with CassandraStressExporter(instance_name=node.cql_ip_address,
metrics=nemesis_metrics_obj(),
stress_operation=stress_cmd_opt,
stress_log_filename=log_file_name,
loader_idx=loader_idx, cpu_idx=cpu_idx), \
CassandraStressEventsPublisher(node=node, cs_log_filename=log_file_name) as publisher, \
CassandraStressEvent(node=node, stress_cmd=self.stress_cmd,
log_file_name=log_file_name) as cs_stress_event:
publisher.event_id = cs_stress_event.event_id
try:
result = node.remoter.run(cmd=node_cmd,
timeout=self.timeout,
log_file=log_file_name)
except Exception as exc: # pylint: disable=broad-except
cs_stress_event.severity = Severity.CRITICAL if self.stop_test_on_failure else Severity.ERROR
cs_stress_event.add_error(
errors=[format_stress_cmd_error(exc)])
return node, result, cs_stress_event
def test_without_errors(self):
event = CassandraStressEvent.error(node=[], stress_cmd="c-s", log_file_name="1.log")
self.assertEqual(event.severity, Severity.ERROR)
self.assertEqual(event.node, "[]")
self.assertEqual(event.stress_cmd, "c-s")
self.assertEqual(event.log_file_name, "1.log")
self.assertIsNone(event.errors)
self.assertEqual(str(event), "(CassandraStressEvent Severity.ERROR): type=error node=[]\nstress_cmd=c-s")
self.assertEqual(event, pickle.loads(pickle.dumps(event)))
def test_with_errors(self):
event = CassandraStressEvent.failure(node="node1", errors=["e1", "e2"])
self.assertEqual(event.severity, Severity.CRITICAL)
self.assertEqual(event.node, "node1")
self.assertIsNone(event.stress_cmd)
self.assertIsNone(event.log_file_name)
self.assertEqual(event.errors, ["e1", "e2"])
self.assertEqual(
str(event),
"(CassandraStressEvent Severity.CRITICAL): type=failure node=node1\ne1\ne2"
)
self.assertEqual(event, pickle.loads(pickle.dumps(event)))
def test_continuous_event_without_errors(self):
begin_event_timestamp = 1623596860.1202102
cs_event = CassandraStressEvent(node="node",
stress_cmd="stress_cmd",
log_file_name="log_file_name",
publish_event=False)
cs_event.event_id = "14f35b64-2fcc-4b6e-a09d-4aeaf4faa543"
begin_event = cs_event.begin_event()
begin_event.event_timestamp = begin_event.begin_timestamp = begin_event_timestamp
self.assertEqual(
str(begin_event),
"(CassandraStressEvent Severity.NORMAL) period_type=begin "
"event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543: node=node\nstress_cmd=stress_cmd"
)
self.assertEqual(begin_event.event_timestamp, begin_event_timestamp)
self.assertEqual(begin_event.timestamp, begin_event_timestamp)
self.assertEqual(begin_event, pickle.loads(pickle.dumps(begin_event)))
end_event_timestamp = 1623596870.1202102
cs_event.end_event()
cs_event.event_timestamp = cs_event.end_timestamp = end_event_timestamp
self.assertEqual(
str(cs_event),
"(CassandraStressEvent Severity.NORMAL) period_type=end "
"event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543 duration=10s: node=node\n"
"stress_cmd=stress_cmd")
self.assertEqual(cs_event.event_timestamp, end_event_timestamp)
self.assertEqual(cs_event.timestamp, end_event_timestamp)
self.assertEqual(cs_event, pickle.loads(pickle.dumps(cs_event)))
def test_continuous_event_with_errors(self):
begin_event_timestamp = 1623596860.1202102
cs_event = CassandraStressEvent(node="node",
stress_cmd="stress_cmd",
log_file_name="log_file_name",
publish_event=False)
cs_event.event_id = "14f35b64-2fcc-4b6e-a09d-4aeaf4faa543"
begin_event = cs_event.begin_event()
begin_event.event_timestamp = begin_event.begin_timestamp = begin_event_timestamp
self.assertEqual(
str(begin_event),
"(CassandraStressEvent Severity.NORMAL) period_type=begin "
"event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543: node=node\nstress_cmd=stress_cmd"
)
self.assertEqual(begin_event.event_timestamp, begin_event_timestamp)
self.assertEqual(begin_event.timestamp, begin_event_timestamp)
self.assertEqual(begin_event, pickle.loads(pickle.dumps(begin_event)))
cs_event.add_error(errors=["error1", "error2"])
cs_event.severity = Severity.ERROR
cs_event.end_event()
end_event_timestamp = 1623596860.1202102
cs_event.event_timestamp = cs_event.end_timestamp = end_event_timestamp
self.assertEqual(
str(cs_event),
"(CassandraStressEvent Severity.ERROR) period_type=end "
"event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543 duration=0s: node=node\nstress_cmd=stress_cmd"
"\nerrors:\n\nerror1\nerror2")
self.assertEqual(cs_event.event_timestamp, end_event_timestamp)
self.assertEqual(cs_event.timestamp, end_event_timestamp)
cs_event.add_error(["One more error"])
cs_event.severity = Severity.CRITICAL
cs_event.event_error()
error_event_timestamp = 1623596860.1202102
cs_event.event_timestamp = error_event_timestamp
self.assertEqual(
str(cs_event),
"(CassandraStressEvent Severity.CRITICAL) period_type=one-time "
"event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543 duration=0s: node=node\n"
"stress_cmd=stress_cmd\nerrors:\n\nerror1\nerror2\nOne more error")
self.assertEqual(cs_event.log_file_name, "log_file_name")
self.assertEqual(cs_event.event_timestamp, error_event_timestamp)
self.assertEqual(cs_event.timestamp, error_event_timestamp)
self.assertEqual(cs_event, pickle.loads(pickle.dumps(cs_event)))