def get_stress_results_bench(self): ret = [] results = [] LOGGER.debug('Wait for stress threads results') for future in concurrent.futures.as_completed(self.results_futures, timeout=self.timeout): results.append(future.result()) for _, result in results: if not result: # Silently skip if stress command throwed error, since it was already reported in _run_stress continue output = result.stdout + result.stderr try: lines = output.splitlines() node_cs_res = self._parse_bench_summary(lines) # pylint: disable=protected-access if node_cs_res: ret.append(node_cs_res) except Exception as exc: ScyllaBenchEvent.error( node="", stress_cmd=self.stress_cmd, errors=[f"Failed to proccess stress summary due to {exc}" ]).publish() return ret
def test_continuous_event_with_error(self): begin_event_timestamp = 1623596860.1202102 scylla_bench_event = ScyllaBenchEvent(node="node", stress_cmd="stress_cmd", log_file_name="log_file_name", publish_event=False) scylla_bench_event.event_id = "14f35b64-2fcc-4b6e-a09d-4aeaf4faa543" begin_event = scylla_bench_event.begin_event() begin_event.timestamp = begin_event.begin_timestamp = begin_event_timestamp self.assertEqual( str(begin_event), "(ScyllaBenchEvent Severity.NORMAL) period_type=begin " "event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543: node=node\nstress_cmd=stress_cmd" ) self.assertEqual(begin_event.timestamp, begin_event_timestamp) self.assertEqual(begin_event, pickle.loads(pickle.dumps(begin_event))) try: raise ValueError('Stress command completed with bad status 1') except Exception as exc: # pylint: disable=broad-except scylla_bench_event.severity = Severity.ERROR scylla_bench_event.add_error([str(exc)]) scylla_bench_event.end_event() scylla_bench_event.end_timestamp = scylla_bench_event.timestamp = 1623596960.1202102 self.assertEqual( str(scylla_bench_event), '(ScyllaBenchEvent Severity.ERROR) period_type=end ' 'event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543 duration=1m40s: ' 'node=node\nstress_cmd=stress_cmd\nerrors:' '\n\nStress command completed with bad status 1') scylla_bench_event.log_file_name = "log_file_name" self.assertEqual(scylla_bench_event, pickle.loads(pickle.dumps(scylla_bench_event)))
def _run_stress_bench(self, node, loader_idx, stress_cmd, node_list): read_gap = 480 # reads starts after write, read can look before start read time to current time using several sstables stress_cmd = re.sub(r"SCT_TIME", f"{int(time.time()) - read_gap}", stress_cmd) LOGGER.debug(f"replaced stress command {stress_cmd}") ScyllaBenchEvent.start(node=node, stress_cmd=stress_cmd).publish() os.makedirs(node.logdir, exist_ok=True) log_file_name = os.path.join( node.logdir, f'scylla-bench-l{loader_idx}-{uuid.uuid4()}.log') # Select first seed node to send the scylla-bench cmds ips = node_list[0].ip_address # Find stress mode: # "scylla-bench -workload=sequential -mode=write -replication-factor=3 -partition-count=100" # "scylla-bench -workload=uniform -mode=read -replication-factor=3 -partition-count=100" found = re.search(r"-mode=(.+?) ", stress_cmd) stress_cmd_opt = found.group(1) with ScyllaBenchStressExporter(instance_name=node.ip_address, metrics=nemesis_metrics_obj(), stress_operation=stress_cmd_opt, stress_log_filename=log_file_name, loader_idx=loader_idx), \ ScyllaBenchStressEventsPublisher(node=node, sb_log_filename=log_file_name): result = None try: result = node.remoter.run( cmd="/$HOME/go/bin/{name} -nodes {ips}".format( name=stress_cmd.strip(), ips=ips), timeout=self.timeout, log_file=log_file_name) except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) if "truncate: seastar::rpc::timeout_error" in errors_str: event_type = ScyllaBenchEvent.timeout elif self.stop_test_on_failure: event_type = ScyllaBenchEvent.failure else: event_type = ScyllaBenchEvent.error event_type( node=node, stress_cmd=stress_cmd, log_file_name=log_file_name, errors=[ errors_str, ], ).publish() else: ScyllaBenchEvent.finish(node=node, stress_cmd=stress_cmd, log_file_name=log_file_name).publish() return node, result
def _run_stress_bench(self, node, loader_idx, stress_cmd, node_list): if self.sb_mode == ScyllaBenchModes.WRITE and self.sb_workload == ScyllaBenchWorkloads.TIMESERIES: node.parent_cluster.sb_write_timeseries_ts = write_timestamp = time.time_ns( ) LOGGER.debug("Set start-time: %s", write_timestamp) stress_cmd = re.sub(r"SET_WRITE_TIMESTAMP", f"{write_timestamp}", stress_cmd) LOGGER.debug("Replaced stress command: %s", stress_cmd) elif self.sb_mode == ScyllaBenchModes.READ and self.sb_workload == ScyllaBenchWorkloads.TIMESERIES: write_timestamp = wait_for( lambda: node.parent_cluster.sb_write_timeseries_ts, step=5, timeout=30, text= 'Waiting for "scylla-bench -workload=timeseries -mode=write" been started, to pick up timestamp' ) LOGGER.debug("Found write timestamp %s", write_timestamp) stress_cmd = re.sub(r"GET_WRITE_TIMESTAMP", f"{write_timestamp}", stress_cmd) LOGGER.debug("replaced stress command %s", stress_cmd) else: LOGGER.debug("Scylla bench command: %s", stress_cmd) os.makedirs(node.logdir, exist_ok=True) log_file_name = os.path.join( node.logdir, f'scylla-bench-l{loader_idx}-{uuid.uuid4()}.log') # Select first seed node to send the scylla-bench cmds ips = node_list[0].cql_ip_address with ScyllaBenchStressExporter(instance_name=node.cql_ip_address, metrics=nemesis_metrics_obj(), stress_operation=self.sb_mode, stress_log_filename=log_file_name, loader_idx=loader_idx), \ ScyllaBenchStressEventsPublisher(node=node, sb_log_filename=log_file_name) as publisher, \ ScyllaBenchEvent(node=node, stress_cmd=self.stress_cmd, log_file_name=log_file_name) as scylla_bench_event: publisher.event_id = scylla_bench_event.event_id result = None try: result = node.remoter.run( cmd="/$HOME/go/bin/{name} -nodes {ips}".format( name=stress_cmd.strip(), ips=ips), timeout=self.timeout, log_file=log_file_name) except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) if "truncate: seastar::rpc::timeout_error" in errors_str: scylla_bench_event.severity = Severity.ERROR elif self.stop_test_on_failure: scylla_bench_event.severity = Severity.CRITICAL else: scylla_bench_event.severity = Severity.ERROR scylla_bench_event.add_error([errors_str]) return node, result
def test_continuous_event_without_error(self): begin_event_timestamp = 1623596860.1202102 scylla_bench_event = ScyllaBenchEvent(node="node", stress_cmd="stress_cmd", log_file_name="log_file_name", publish_event=False) scylla_bench_event.event_id = "14f35b64-2fcc-4b6e-a09d-4aeaf4faa543" begin_event = scylla_bench_event.begin_event() begin_event.event_timestamp = begin_event.begin_timestamp = begin_event_timestamp self.assertEqual( str(begin_event), "(ScyllaBenchEvent Severity.NORMAL) period_type=begin " "event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543: node=node\nstress_cmd=stress_cmd" ) self.assertEqual(begin_event.event_timestamp, begin_event_timestamp) self.assertEqual(begin_event.timestamp, begin_event_timestamp) self.assertEqual(begin_event, pickle.loads(pickle.dumps(begin_event))) scylla_bench_event.end_event() scylla_bench_event.event_timestamp = scylla_bench_event.end_timestamp = 1623596861.1202102 self.assertEqual( str(scylla_bench_event), '(ScyllaBenchEvent Severity.NORMAL) period_type=end ' 'event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543 duration=1s: ' 'node=node\nstress_cmd=stress_cmd') scylla_bench_event.log_file_name = "log_file_name" self.assertEqual(scylla_bench_event, pickle.loads(pickle.dumps(scylla_bench_event)))
def test_without_errors(self): event = ScyllaBenchEvent.timeout(node=[], stress_cmd="c-s", log_file_name="1.log") self.assertEqual(event.severity, Severity.ERROR) self.assertEqual(event.node, "[]") self.assertEqual(event.stress_cmd, "c-s") self.assertEqual(event.log_file_name, "1.log") self.assertIsNone(event.errors) self.assertEqual(str(event), "(ScyllaBenchEvent Severity.ERROR): type=timeout node=[] stress_cmd=c-s") self.assertEqual(event, pickle.loads(pickle.dumps(event)))
def test_with_errors(self): event = ScyllaBenchEvent.failure(node="node1", errors=["e1", "e2"]) self.assertEqual(event.severity, Severity.CRITICAL) self.assertEqual(event.node, "node1") self.assertIsNone(event.stress_cmd) self.assertIsNone(event.log_file_name) self.assertEqual(event.errors, ["e1", "e2"]) self.assertEqual( str(event), "(ScyllaBenchEvent Severity.CRITICAL): type=failure node=node1 stress_cmd=None error=e1\ne2" ) self.assertEqual(event, pickle.loads(pickle.dumps(event)))