def _run_gemini(self, node, loader_idx): logdir = os.path.join(self.outputdir, node.name) makedirs(logdir) log_file_name = os.path.join(logdir, 'gemini-l%s-%s.log' % (loader_idx, uuid.uuid4())) gemini_cmd = self._generate_gemini_command(loader_idx) GeminiEvent(type='start', cmd=gemini_cmd) try: with GeminiEventsPublisher(node=node, gemini_log_filename=log_file_name): result = node.remoter.run(cmd=gemini_cmd, timeout=self.timeout, ignore_status=False, log_file=log_file_name) # sleep to gather all latest log messages time.sleep(5) except Exception as details: # pylint: disable=broad-except LOGGER.error(details) result = getattr(details, "result", NotGeminiErrorResult(details)) GeminiEvent(type='finish', cmd=gemini_cmd, result={'exit_code': result.exited, 'stdout': result.stdout, 'stderr': result.stderr}) return node, result, self.gemini_log
def _run_stress(self, node, loader_idx, cpu_idx, keyspace_idx): stress_cmd = self.create_stress_cmd(node, loader_idx, keyspace_idx) if self.profile: with open(self.profile) as profile_file: LOGGER.info('Profile content:\n%s', profile_file.read()) node.remoter.send_files(self.profile, os.path.join( '/tmp', os.path.basename(self.profile)), delete_dst=True) stress_cmd_opt = stress_cmd.split()[1] LOGGER.info('Stress command:\n%s', stress_cmd) log_dir = os.path.join(self.output_dir, self.loader_set.name) if not os.path.exists(log_dir): makedirs(log_dir) log_file_name = os.path.join( log_dir, f'cassandra-stress-l{loader_idx}-c{cpu_idx}-k{keyspace_idx}-{uuid.uuid4()}.log' ) LOGGER.debug('cassandra-stress local log: %s', log_file_name) # This tag will be output in the header of c-stress result, # we parse it to know the loader & cpu info in _parse_cs_summary(). tag = f'TAG: loader_idx:{loader_idx}-cpu_idx:{cpu_idx}-keyspace_idx:{keyspace_idx}' if self.stress_num > 1: node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; taskset -c {cpu_idx} {stress_cmd}' else: node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {stress_cmd}' node_cmd = f'echo {tag}; {node_cmd}' CassandraStressEvent(type='start', node=str(node), stress_cmd=stress_cmd) with CassandraStressExporter(instance_name=node.ip_address, metrics=nemesis_metrics_obj(), cs_operation=stress_cmd_opt, cs_log_filename=log_file_name, loader_idx=loader_idx, cpu_idx=cpu_idx), \ CassandraStressEventsPublisher(node=node, cs_log_filename=log_file_name): result = node.remoter.run(cmd=node_cmd, timeout=self.timeout, ignore_status=True, log_file=log_file_name) CassandraStressEvent(type='finish', node=str(node), stress_cmd=stress_cmd, log_file_name=log_file_name) return node, result
def __init__(self, log_dir): super(EventsDevice, self).__init__() self.ready_event = Event() self.pub_port = Value('d', 0) self.sub_port = Value('d', 0) self.event_log_base_dir = os.path.join(log_dir, 'events_log') makedirs(self.event_log_base_dir) self.raw_events_filename = os.path.join(self.event_log_base_dir, 'raw_events.log')
def __init__(self, name, node_prefix=None, parent_cluster=None, base_logdir=None): # pylint: disable=too-many-arguments,super-init-not-called self.name = name self.node_prefix = node_prefix self.remoter = LOCALRUNNER self.remoter.receive_files = types.MethodType(send_receive_files, self) self.remoter.send_files = types.MethodType(send_receive_files, self) self.parent_cluster = parent_cluster self.is_seed = False self._distro = None self.logdir = os.path.join(base_logdir, self.name) makedirs(self.logdir) self.log = SDCMAdapter(LOGGER, extra={'prefix': str(self)})
def collect(self, node, local_dst, remote_dst=None, local_search_path=None): makedirs(local_dst) if self.search_locally and local_search_path: search_pattern = self.name if not node else "/".join( [node.name, self.name]) local_logfiles = self.find_local_files(local_search_path, search_pattern) for logfile in local_logfiles: shutil.copy(src=logfile, dst=local_dst) if self.cmd and not self._is_file_collected(local_dst): super(FileLog, self).collect(node, local_dst, remote_dst) return local_dst
def receive_log(node, remote_log_path, local_dir): makedirs(local_dir) if node.remoter: node.remoter.receive_files(src=remote_log_path, dst=local_dir) return local_dir
def _run_stress(self, node, loader_idx, cpu_idx, keyspace_idx): # pylint: disable=too-many-locals stress_cmd = self.create_stress_cmd(node, loader_idx, keyspace_idx) if self.profile: with open(self.profile) as profile_file: LOGGER.info('Profile content:\n%s', profile_file.read()) node.remoter.send_files(self.profile, os.path.join( '/tmp', os.path.basename(self.profile)), delete_dst=True) stress_cmd_opt = stress_cmd.split()[1] LOGGER.info('Stress command:\n%s', stress_cmd) if not os.path.exists(node.logdir): makedirs(node.logdir) log_file_name = os.path.join( node.logdir, f'cassandra-stress-l{loader_idx}-c{cpu_idx}-k{keyspace_idx}-{uuid.uuid4()}.log' ) LOGGER.debug('cassandra-stress local log: %s', log_file_name) # This tag will be output in the header of c-stress result, # we parse it to know the loader & cpu info in _parse_cs_summary(). tag = f'TAG: loader_idx:{loader_idx}-cpu_idx:{cpu_idx}-keyspace_idx:{keyspace_idx}' if self.stress_num > 1: node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; taskset -c {cpu_idx} {stress_cmd}' else: node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {stress_cmd}' node_cmd = f'echo {tag}; {node_cmd}' CassandraStressEvent(type='start', node=str(node), stress_cmd=stress_cmd) with CassandraStressExporter(instance_name=node.ip_address, metrics=nemesis_metrics_obj(), cs_operation=stress_cmd_opt, cs_log_filename=log_file_name, loader_idx=loader_idx, cpu_idx=cpu_idx), \ CassandraStressEventsPublisher(node=node, cs_log_filename=log_file_name): result = None try: result = node.remoter.run(cmd=node_cmd, timeout=self.timeout, log_file=log_file_name) except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) event_type = 'failure' if self.stop_test_on_failure else 'error' event_severity = Severity.CRITICAL if self.stop_test_on_failure else Severity.ERROR CassandraStressEvent(type=event_type, node=str(node), stress_cmd=stress_cmd, log_file_name=log_file_name, severity=event_severity, errors=[errors_str]) CassandraStressEvent(type='finish', node=str(node), stress_cmd=stress_cmd, log_file_name=log_file_name) return node, result