def get_results(self): ret = [] results = [] LOGGER.debug('Wait for %s stress threads results', self.max_workers) for future in concurrent.futures.as_completed(self.results_futures, timeout=self.timeout): results.append(future.result()) for _, result in results: if not result: # Silently skip if stress command threw error, since it was already reported in _run_stress continue output = result.stdout + result.stderr try: lines = output.splitlines() node_cs_res = BaseLoaderSet._parse_cs_summary(lines) # pylint: disable=protected-access if node_cs_res: ret.append(node_cs_res) except Exception as exc: # pylint: disable=broad-except CassandraStressEvent.failure( node="", errors=[ f"Failed to process stress summary due to {exc}", ]) return ret
def _run_stress(self, node, loader_idx, cpu_idx, keyspace_idx): # pylint: disable=too-many-locals stress_cmd = self.create_stress_cmd(node, loader_idx, keyspace_idx) if self.profile: with open(self.profile) as profile_file: LOGGER.info('Profile content:\n%s', profile_file.read()) node.remoter.send_files(self.profile, os.path.join( '/tmp', os.path.basename(self.profile)), delete_dst=True) stress_cmd_opt = stress_cmd.split()[1] LOGGER.info('Stress command:\n%s', stress_cmd) os.makedirs(node.logdir, exist_ok=True) log_file_name = \ os.path.join(node.logdir, f'cassandra-stress-l{loader_idx}-c{cpu_idx}-k{keyspace_idx}-{uuid.uuid4()}.log') LOGGER.debug('cassandra-stress local log: %s', log_file_name) # This tag will be output in the header of c-stress result, # we parse it to know the loader & cpu info in _parse_cs_summary(). tag = f'TAG: loader_idx:{loader_idx}-cpu_idx:{cpu_idx}-keyspace_idx:{keyspace_idx}' if self.stress_num > 1: node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; taskset -c {cpu_idx} {stress_cmd}' else: node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {stress_cmd}' node_cmd = f'echo {tag}; {node_cmd}' result = None CassandraStressEvent.start(node=node, stress_cmd=stress_cmd).publish() with CassandraStressExporter(instance_name=node.ip_address, metrics=nemesis_metrics_obj(), stress_operation=stress_cmd_opt, stress_log_filename=log_file_name, loader_idx=loader_idx, cpu_idx=cpu_idx), \ CassandraStressEventsPublisher(node=node, cs_log_filename=log_file_name): try: result = node.remoter.run(cmd=node_cmd, timeout=self.timeout, log_file=log_file_name) except Exception as exc: event_type = CassandraStressEvent.failure if self.stop_test_on_failure else CassandraStressEvent.error event_type(node=node, stress_cmd=stress_cmd, log_file_name=log_file_name, errors=[ format_stress_cmd_error(exc), ]).publish() CassandraStressEvent.finish(node=node, stress_cmd=stress_cmd, log_file_name=log_file_name).publish() return node, result
def _run_stress(self, node, loader_idx, cpu_idx, keyspace_idx): # pylint: disable=too-many-locals stress_cmd = self.create_stress_cmd(node, loader_idx, keyspace_idx) if self.profile: with open(self.profile, encoding="utf-8") as profile_file: LOGGER.info('Profile content:\n%s', profile_file.read()) node.remoter.send_files(self.profile, os.path.join( '/tmp', os.path.basename(self.profile)), delete_dst=True) # Get next word after `cassandra-stress' in stress_cmd. # Do it this way because stress_cmd can contain env variables before `cassandra-stress'. stress_cmd_opt = stress_cmd.split("cassandra-stress", 1)[1].split(None, 1)[0] LOGGER.info('Stress command:\n%s', stress_cmd) os.makedirs(node.logdir, exist_ok=True) log_file_name = \ os.path.join(node.logdir, f'cassandra-stress-l{loader_idx}-c{cpu_idx}-k{keyspace_idx}-{uuid.uuid4()}.log') LOGGER.debug('cassandra-stress local log: %s', log_file_name) # This tag will be output in the header of c-stress result, # we parse it to know the loader & cpu info in _parse_cs_summary(). tag = f'TAG: loader_idx:{loader_idx}-cpu_idx:{cpu_idx}-keyspace_idx:{keyspace_idx}' if self.stress_num > 1: node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; taskset -c {cpu_idx} {stress_cmd}' else: node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {stress_cmd}' node_cmd = f'echo {tag}; {node_cmd}' result = None # disable logging for cassandra stress node.remoter.run("cp /etc/scylla/cassandra/logback-tools.xml .", ignore_status=True) with CassandraStressExporter(instance_name=node.cql_ip_address, metrics=nemesis_metrics_obj(), stress_operation=stress_cmd_opt, stress_log_filename=log_file_name, loader_idx=loader_idx, cpu_idx=cpu_idx), \ CassandraStressEventsPublisher(node=node, cs_log_filename=log_file_name) as publisher, \ CassandraStressEvent(node=node, stress_cmd=self.stress_cmd, log_file_name=log_file_name) as cs_stress_event: publisher.event_id = cs_stress_event.event_id try: result = node.remoter.run(cmd=node_cmd, timeout=self.timeout, log_file=log_file_name) except Exception as exc: # pylint: disable=broad-except cs_stress_event.severity = Severity.CRITICAL if self.stop_test_on_failure else Severity.ERROR cs_stress_event.add_error( errors=[format_stress_cmd_error(exc)]) return node, result, cs_stress_event
def test_without_errors(self): event = CassandraStressEvent.error(node=[], stress_cmd="c-s", log_file_name="1.log") self.assertEqual(event.severity, Severity.ERROR) self.assertEqual(event.node, "[]") self.assertEqual(event.stress_cmd, "c-s") self.assertEqual(event.log_file_name, "1.log") self.assertIsNone(event.errors) self.assertEqual(str(event), "(CassandraStressEvent Severity.ERROR): type=error node=[]\nstress_cmd=c-s") self.assertEqual(event, pickle.loads(pickle.dumps(event)))
def test_with_errors(self): event = CassandraStressEvent.failure(node="node1", errors=["e1", "e2"]) self.assertEqual(event.severity, Severity.CRITICAL) self.assertEqual(event.node, "node1") self.assertIsNone(event.stress_cmd) self.assertIsNone(event.log_file_name) self.assertEqual(event.errors, ["e1", "e2"]) self.assertEqual( str(event), "(CassandraStressEvent Severity.CRITICAL): type=failure node=node1\ne1\ne2" ) self.assertEqual(event, pickle.loads(pickle.dumps(event)))
def test_continuous_event_without_errors(self): begin_event_timestamp = 1623596860.1202102 cs_event = CassandraStressEvent(node="node", stress_cmd="stress_cmd", log_file_name="log_file_name", publish_event=False) cs_event.event_id = "14f35b64-2fcc-4b6e-a09d-4aeaf4faa543" begin_event = cs_event.begin_event() begin_event.event_timestamp = begin_event.begin_timestamp = begin_event_timestamp self.assertEqual( str(begin_event), "(CassandraStressEvent Severity.NORMAL) period_type=begin " "event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543: node=node\nstress_cmd=stress_cmd" ) self.assertEqual(begin_event.event_timestamp, begin_event_timestamp) self.assertEqual(begin_event.timestamp, begin_event_timestamp) self.assertEqual(begin_event, pickle.loads(pickle.dumps(begin_event))) end_event_timestamp = 1623596870.1202102 cs_event.end_event() cs_event.event_timestamp = cs_event.end_timestamp = end_event_timestamp self.assertEqual( str(cs_event), "(CassandraStressEvent Severity.NORMAL) period_type=end " "event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543 duration=10s: node=node\n" "stress_cmd=stress_cmd") self.assertEqual(cs_event.event_timestamp, end_event_timestamp) self.assertEqual(cs_event.timestamp, end_event_timestamp) self.assertEqual(cs_event, pickle.loads(pickle.dumps(cs_event)))
def test_continuous_event_with_errors(self): begin_event_timestamp = 1623596860.1202102 cs_event = CassandraStressEvent(node="node", stress_cmd="stress_cmd", log_file_name="log_file_name", publish_event=False) cs_event.event_id = "14f35b64-2fcc-4b6e-a09d-4aeaf4faa543" begin_event = cs_event.begin_event() begin_event.event_timestamp = begin_event.begin_timestamp = begin_event_timestamp self.assertEqual( str(begin_event), "(CassandraStressEvent Severity.NORMAL) period_type=begin " "event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543: node=node\nstress_cmd=stress_cmd" ) self.assertEqual(begin_event.event_timestamp, begin_event_timestamp) self.assertEqual(begin_event.timestamp, begin_event_timestamp) self.assertEqual(begin_event, pickle.loads(pickle.dumps(begin_event))) cs_event.add_error(errors=["error1", "error2"]) cs_event.severity = Severity.ERROR cs_event.end_event() end_event_timestamp = 1623596860.1202102 cs_event.event_timestamp = cs_event.end_timestamp = end_event_timestamp self.assertEqual( str(cs_event), "(CassandraStressEvent Severity.ERROR) period_type=end " "event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543 duration=0s: node=node\nstress_cmd=stress_cmd" "\nerrors:\n\nerror1\nerror2") self.assertEqual(cs_event.event_timestamp, end_event_timestamp) self.assertEqual(cs_event.timestamp, end_event_timestamp) cs_event.add_error(["One more error"]) cs_event.severity = Severity.CRITICAL cs_event.event_error() error_event_timestamp = 1623596860.1202102 cs_event.event_timestamp = error_event_timestamp self.assertEqual( str(cs_event), "(CassandraStressEvent Severity.CRITICAL) period_type=one-time " "event_id=14f35b64-2fcc-4b6e-a09d-4aeaf4faa543 duration=0s: node=node\n" "stress_cmd=stress_cmd\nerrors:\n\nerror1\nerror2\nOne more error") self.assertEqual(cs_event.log_file_name, "log_file_name") self.assertEqual(cs_event.event_timestamp, error_event_timestamp) self.assertEqual(cs_event.timestamp, error_event_timestamp) self.assertEqual(cs_event, pickle.loads(pickle.dumps(cs_event)))