def test_ycsb_filter(self):
with self.wait_for_n_events(self.get_events_logger(), count=4, timeout=3):
with EventsFilter(event_class=YcsbStressEvent,
regex=".*Internal server error: exceptions::unavailable_exception.*"):
YcsbStressEvent.error(
node="Node alternator-3h-silence--loader-node-bb90aa05-2"
" [34.251.153.122 | 10.0.220.55] (seed: False)",
stress_cmd="ycsb",
errors=["237951 [Thread-47] ERROR site.ycsb.db.DynamoDBClient"
" -com.amazonaws.AmazonServiceException: Internal server error:"
" exceptions::unavailable_exception (Cannot achieve consistency"
" level for cl LOCAL_ONE. Requires 1, alive 0) (Service: AmazonDynamoDBv2;"
" Status Code: 500; Error Code: Internal Server Error; Request ID: null)"]).publish()
TestFrameworkEvent(source="", source_method="").publish()
log_content = self.get_event_log_file("events.log")
self.assertIn("TestFrameworkEvent", log_content)
self.assertNotIn("YcsbStressEvent", log_content)
with self.wait_for_n_events(self.get_events_logger(), count=1):
YcsbStressEvent.error(
node="Node alternator-3h-silence--loader-node-bb90aa05-2"
" [34.251.153.122 | 10.0.220.55] (seed: False)",
stress_cmd="ycsb",
errors=["237951 [Thread-47] ERROR site.ycsb.db.DynamoDBClient"
" -com.amazonaws.AmazonServiceException: Internal server error:"
" exceptions::unavailable_exception (Cannot achieve consistency"
" level for cl LOCAL_ONE. Requires 1, alive 0) (Service: AmazonDynamoDBv2;"
" Status Code: 500; Error Code: Internal Server Error; Request ID: null)"]).publish()
log_content = self.get_event_log_file("events.log")
self.assertIn("TestFrameworkEvent", log_content)
self.assertIn("YcsbStressEvent", log_content)
def raise_event_callback(sentinel, line): # pylint: disable=unused-argument
if line:
YcsbStressEvent.error(node=loader,
stress_cmd=stress_cmd,
errors=[
line,
]).publish()
def test_without_errors(self):
event = YcsbStressEvent.error(node=[], stress_cmd="c-s", log_file_name="1.log")
self.assertEqual(event.severity, Severity.ERROR)
self.assertEqual(event.node, "[]")
self.assertEqual(event.stress_cmd, "c-s")
self.assertEqual(event.log_file_name, "1.log")
self.assertIsNone(event.errors)
self.assertEqual(str(event), "(YcsbStressEvent Severity.ERROR): type=error node=[]\nstress_cmd=c-s")
self.assertEqual(event, pickle.loads(pickle.dumps(event)))
def test_with_errors(self):
event = YcsbStressEvent.failure(node="node1", errors=["e1", "e2"])
self.assertEqual(event.severity, Severity.CRITICAL)
self.assertEqual(event.node, "node1")
self.assertIsNone(event.stress_cmd)
self.assertIsNone(event.log_file_name)
self.assertEqual(event.errors, ["e1", "e2"])
self.assertEqual(
str(event),
"(YcsbStressEvent Severity.CRITICAL): type=failure node=node1\nstress_cmd=None\nerrors:\n\ne1\ne2"
)
self.assertEqual(event, pickle.loads(pickle.dumps(event)))
def test_without_errors(self):
event = YcsbStressEvent.error(node=[], stress_cmd="c-s", log_file_name="1.log")
self.assertEqual(event.severity, Severity.ERROR)
self.assertEqual(event.node, "[]")
self.assertEqual(event.stress_cmd, "c-s")
self.assertEqual(event.log_file_name, "1.log")
self.assertIsNone(event.errors)
event.event_id = "68067fe2-4c9e-421c-97b5-12db8d7ba71d"
self.assertEqual(str(event),
"(YcsbStressEvent Severity.ERROR) period_type=not-set "
"event_id=68067fe2-4c9e-421c-97b5-12db8d7ba71d: type=error node=[]\nstress_cmd=c-s")
self.assertEqual(event, pickle.loads(pickle.dumps(event)))
def test_with_errors(self):
event = YcsbStressEvent.failure(node="node1", errors=["e1", "e2"])
self.assertEqual(event.severity, Severity.CRITICAL)
self.assertEqual(event.node, "node1")
self.assertIsNone(event.stress_cmd)
self.assertIsNone(event.log_file_name)
self.assertEqual(event.errors, ["e1", "e2"])
event.event_id = "225676a7-ddd1-4f4d-bae8-1cf5b35d0955"
self.assertEqual(
str(event),
"(YcsbStressEvent Severity.CRITICAL) period_type=not-set "
"event_id=225676a7-ddd1-4f4d-bae8-1cf5b35d0955:"
" type=failure node=node1\nerrors:\n\ne1\ne2")
self.assertEqual(event, pickle.loads(pickle.dumps(event)))
def _run_stress(self, loader, loader_idx, cpu_idx):
dns_options = ""
cpu_options = ""
if self.params.get('alternator_use_dns_routing'):
dns = RemoteDocker(
loader,
"scylladb/hydra-loaders:alternator-dns-0.2",
command_line=
f'python3 /dns_server.py {self.db_node_to_query(loader)} '
f'{self.params.get("alternator_port")}',
extra_docker_opts=f'--label shell_marker={self.shell_marker}')
dns_options += f'--dns {dns.internal_ip_address} --dns-option use-vc'
if self.stress_num > 1:
cpu_options = '--cpuset-cpus="{cpu_idx}"'
docker = RemoteDocker(
loader,
"scylladb/hydra-loaders:ycsb-jdk8-20211104",
extra_docker_opts=
f'{dns_options} {cpu_options} --label shell_marker={self.shell_marker}'
)
self.copy_template(docker)
stress_cmd = self.build_stress_cmd()
if not os.path.exists(loader.logdir):
os.makedirs(loader.logdir, exist_ok=True)
log_file_name = os.path.join(
loader.logdir,
'ycsb-l%s-c%s-%s.log' % (loader_idx, cpu_idx, uuid.uuid4()))
LOGGER.debug('ycsb-stress local log: %s', log_file_name)
def raise_event_callback(sentinel, line): # pylint: disable=unused-argument
if line:
YcsbStressEvent.error(node=loader,
stress_cmd=stress_cmd,
errors=[
line,
]).publish()
LOGGER.debug("running: %s", stress_cmd)
node_cmd = 'cd /YCSB && {}'.format(stress_cmd)
YcsbStressEvent.start(node=loader, stress_cmd=stress_cmd).publish()
with YcsbStatsPublisher(loader,
loader_idx,
ycsb_log_filename=log_file_name):
try:
result = docker.run(
cmd=node_cmd,
timeout=self.timeout + self.shutdown_timeout,
log_file=log_file_name,
watchers=[
FailuresWatcher(r'\sERROR|=UNEXPECTED_STATE|=ERROR',
callback=raise_event_callback,
raise_exception=False)
])
return self.parse_final_output(result)
except Exception as exc:
errors_str = format_stress_cmd_error(exc)
YcsbStressEvent.failure(
node=loader,
stress_cmd=self.stress_cmd,
log_file_name=log_file_name,
errors=[
errors_str,
],
).publish()
raise
finally:
YcsbStressEvent.finish(node=loader,
stress_cmd=stress_cmd,
log_file_name=log_file_name).publish()
