def get_stress_results_bench(self):
ret = []
results = []
LOGGER.debug('Wait for stress threads results')
for future in concurrent.futures.as_completed(self.results_futures,
timeout=self.timeout):
results.append(future.result())
for _, result in results:
if not result:
# Silently skip if stress command throwed error, since it was already reported in _run_stress
continue
output = result.stdout + result.stderr
try:
lines = output.splitlines()
node_cs_res = self._parse_bench_summary(lines) # pylint: disable=protected-access
if node_cs_res:
ret.append(node_cs_res)
except Exception as exc:
ScyllaBenchEvent.error(
node="",
stress_cmd=self.stress_cmd,
errors=[f"Failed to proccess stress summary due to {exc}"
]).publish()
return ret
def test_continuous_event_with_error(self):
begin_event_timestamp = 1623596860.1202102
scylla_bench_event = ScyllaBenchEvent(node="node",
stress_cmd="stress_cmd",
log_file_name="log_file_name",
publish_event=False)
scylla_bench_event.event_id = "14f35b64-2fcc-4b6e-a09d-4aeaf4faa543"
begin_event = scylla_bench_event.begin_event()
begin_event.timestamp = begin_event.begin_timestamp = begin_event_timestamp
self.assertEqual(
str(begin_event),
"(ScyllaBenchEvent Severity.NORMAL) period_type=begin "
"event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543: node=node\nstress_cmd=stress_cmd"
)
self.assertEqual(begin_event.timestamp, begin_event_timestamp)
self.assertEqual(begin_event, pickle.loads(pickle.dumps(begin_event)))
try:
raise ValueError('Stress command completed with bad status 1')
except Exception as exc: # pylint: disable=broad-except
scylla_bench_event.severity = Severity.ERROR
scylla_bench_event.add_error([str(exc)])
scylla_bench_event.end_event()
scylla_bench_event.end_timestamp = scylla_bench_event.timestamp = 1623596960.1202102
self.assertEqual(
str(scylla_bench_event),
'(ScyllaBenchEvent Severity.ERROR) period_type=end '
'event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543 duration=1m40s: '
'node=node\nstress_cmd=stress_cmd\nerrors:'
'\n\nStress command completed with bad status 1')
scylla_bench_event.log_file_name = "log_file_name"
self.assertEqual(scylla_bench_event,
pickle.loads(pickle.dumps(scylla_bench_event)))
def _run_stress_bench(self, node, loader_idx, stress_cmd, node_list):
read_gap = 480 # reads starts after write, read can look before start read time to current time using several sstables
stress_cmd = re.sub(r"SCT_TIME", f"{int(time.time()) - read_gap}",
stress_cmd)
LOGGER.debug(f"replaced stress command {stress_cmd}")
ScyllaBenchEvent.start(node=node, stress_cmd=stress_cmd).publish()
os.makedirs(node.logdir, exist_ok=True)
log_file_name = os.path.join(
node.logdir, f'scylla-bench-l{loader_idx}-{uuid.uuid4()}.log')
# Select first seed node to send the scylla-bench cmds
ips = node_list[0].ip_address
# Find stress mode:
# "scylla-bench -workload=sequential -mode=write -replication-factor=3 -partition-count=100"
# "scylla-bench -workload=uniform -mode=read -replication-factor=3 -partition-count=100"
found = re.search(r"-mode=(.+?) ", stress_cmd)
stress_cmd_opt = found.group(1)
with ScyllaBenchStressExporter(instance_name=node.ip_address,
metrics=nemesis_metrics_obj(),
stress_operation=stress_cmd_opt,
stress_log_filename=log_file_name,
loader_idx=loader_idx), \
ScyllaBenchStressEventsPublisher(node=node, sb_log_filename=log_file_name):
result = None
try:
result = node.remoter.run(
cmd="/$HOME/go/bin/{name} -nodes {ips}".format(
name=stress_cmd.strip(), ips=ips),
timeout=self.timeout,
log_file=log_file_name)
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
if "truncate: seastar::rpc::timeout_error" in errors_str:
event_type = ScyllaBenchEvent.timeout
elif self.stop_test_on_failure:
event_type = ScyllaBenchEvent.failure
else:
event_type = ScyllaBenchEvent.error
event_type(
node=node,
stress_cmd=stress_cmd,
log_file_name=log_file_name,
errors=[
errors_str,
],
).publish()
else:
ScyllaBenchEvent.finish(node=node,
stress_cmd=stress_cmd,
log_file_name=log_file_name).publish()
return node, result
def _run_stress_bench(self, node, loader_idx, stress_cmd, node_list):
if self.sb_mode == ScyllaBenchModes.WRITE and self.sb_workload == ScyllaBenchWorkloads.TIMESERIES:
node.parent_cluster.sb_write_timeseries_ts = write_timestamp = time.time_ns(
)
LOGGER.debug("Set start-time: %s", write_timestamp)
stress_cmd = re.sub(r"SET_WRITE_TIMESTAMP", f"{write_timestamp}",
stress_cmd)
LOGGER.debug("Replaced stress command: %s", stress_cmd)
elif self.sb_mode == ScyllaBenchModes.READ and self.sb_workload == ScyllaBenchWorkloads.TIMESERIES:
write_timestamp = wait_for(
lambda: node.parent_cluster.sb_write_timeseries_ts,
step=5,
timeout=30,
text=
'Waiting for "scylla-bench -workload=timeseries -mode=write" been started, to pick up timestamp'
)
LOGGER.debug("Found write timestamp %s", write_timestamp)
stress_cmd = re.sub(r"GET_WRITE_TIMESTAMP", f"{write_timestamp}",
stress_cmd)
LOGGER.debug("replaced stress command %s", stress_cmd)
else:
LOGGER.debug("Scylla bench command: %s", stress_cmd)
os.makedirs(node.logdir, exist_ok=True)
log_file_name = os.path.join(
node.logdir, f'scylla-bench-l{loader_idx}-{uuid.uuid4()}.log')
# Select first seed node to send the scylla-bench cmds
ips = node_list[0].cql_ip_address
with ScyllaBenchStressExporter(instance_name=node.cql_ip_address,
metrics=nemesis_metrics_obj(),
stress_operation=self.sb_mode,
stress_log_filename=log_file_name,
loader_idx=loader_idx), \
ScyllaBenchStressEventsPublisher(node=node, sb_log_filename=log_file_name) as publisher, \
ScyllaBenchEvent(node=node, stress_cmd=self.stress_cmd,
log_file_name=log_file_name) as scylla_bench_event:
publisher.event_id = scylla_bench_event.event_id
result = None
try:
result = node.remoter.run(
cmd="/$HOME/go/bin/{name} -nodes {ips}".format(
name=stress_cmd.strip(), ips=ips),
timeout=self.timeout,
log_file=log_file_name)
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
if "truncate: seastar::rpc::timeout_error" in errors_str:
scylla_bench_event.severity = Severity.ERROR
elif self.stop_test_on_failure:
scylla_bench_event.severity = Severity.CRITICAL
else:
scylla_bench_event.severity = Severity.ERROR
scylla_bench_event.add_error([errors_str])
return node, result
def test_continuous_event_without_error(self):
begin_event_timestamp = 1623596860.1202102
scylla_bench_event = ScyllaBenchEvent(node="node",
stress_cmd="stress_cmd",
log_file_name="log_file_name",
publish_event=False)
scylla_bench_event.event_id = "14f35b64-2fcc-4b6e-a09d-4aeaf4faa543"
begin_event = scylla_bench_event.begin_event()
begin_event.event_timestamp = begin_event.begin_timestamp = begin_event_timestamp
self.assertEqual(
str(begin_event),
"(ScyllaBenchEvent Severity.NORMAL) period_type=begin "
"event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543: node=node\nstress_cmd=stress_cmd"
)
self.assertEqual(begin_event.event_timestamp, begin_event_timestamp)
self.assertEqual(begin_event.timestamp, begin_event_timestamp)
self.assertEqual(begin_event, pickle.loads(pickle.dumps(begin_event)))
scylla_bench_event.end_event()
scylla_bench_event.event_timestamp = scylla_bench_event.end_timestamp = 1623596861.1202102
self.assertEqual(
str(scylla_bench_event),
'(ScyllaBenchEvent Severity.NORMAL) period_type=end '
'event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543 duration=1s: '
'node=node\nstress_cmd=stress_cmd')
scylla_bench_event.log_file_name = "log_file_name"
self.assertEqual(scylla_bench_event,
pickle.loads(pickle.dumps(scylla_bench_event)))
def test_without_errors(self):
event = ScyllaBenchEvent.timeout(node=[], stress_cmd="c-s", log_file_name="1.log")
self.assertEqual(event.severity, Severity.ERROR)
self.assertEqual(event.node, "[]")
self.assertEqual(event.stress_cmd, "c-s")
self.assertEqual(event.log_file_name, "1.log")
self.assertIsNone(event.errors)
self.assertEqual(str(event), "(ScyllaBenchEvent Severity.ERROR): type=timeout node=[] stress_cmd=c-s")
self.assertEqual(event, pickle.loads(pickle.dumps(event)))
def test_with_errors(self):
event = ScyllaBenchEvent.failure(node="node1", errors=["e1", "e2"])
self.assertEqual(event.severity, Severity.CRITICAL)
self.assertEqual(event.node, "node1")
self.assertIsNone(event.stress_cmd)
self.assertIsNone(event.log_file_name)
self.assertEqual(event.errors, ["e1", "e2"])
self.assertEqual(
str(event),
"(ScyllaBenchEvent Severity.CRITICAL): type=failure node=node1 stress_cmd=None error=e1\ne2"
)
self.assertEqual(event, pickle.loads(pickle.dumps(event)))