def test_high_pending_mutations(self):
"""verify StageAnalyzer makes recs on high pending local writes"""
analyzer = recs.Engine(None, None)
stage = recs.Stage(
name="MutationStage",
pending=10001,
active=0,
local_backpressure=0,
completed=0,
blocked=0,
all_time_blocked=0,
)
reason, rec = analyzer.analyze_stage(stage)
self.assertEqual(rec, "lower memtable_cleanup_threshold in cassandra.yaml")
self.assertEqual(reason, "mutations pending over 10000")
stage.pending = 9999
reason, rec = analyzer.analyze_stage(stage)
self.assertIsNone(rec)
self.assertIsNone(reason)
def test_memtable_flush_writer_pending(self):
"""verify flush writer pending"""
analyzer = recs.Engine()
stage = recs.Stage(
name="MemtableFlushWriter",
pending=6,
active=0,
local_backpressure=0,
completed=0,
blocked=0,
all_time_blocked=0,
)
reason, rec = analyzer.analyze_stage(stage)
self.assertEqual(rec, "raise memtable_flush_writers in cassandra.yaml")
self.assertEqual(reason, "memtable flush writers pending over 5")
stage.pending = 0
reason, rec = analyzer.analyze_stage(stage)
self.assertIsNone(rec)
self.assertIsNone(reason)
def test_memtable_flush_writer_blocked(self):
"""verify flush writer blocked"""
analyzer = recs.Engine(None, None)
stage = recs.Stage(
name="MemtableFlushWriter",
pending=0,
active=0,
local_backpressure=0,
completed=0,
blocked=1,
all_time_blocked=0,
)
reason, rec = analyzer.analyze_stage(stage)
self.assertEqual(rec, "lower memtable_cleanup_threshold in cassandra.yaml")
self.assertEqual(reason, "memtable flush writers blocked greater than zero")
stage.blocked = 0
reason, rec = analyzer.analyze_stage(stage)
self.assertIsNone(rec)
self.assertIsNone(reason)
def test_high_pending_write_remote(self):
"""verify StageAnalyzer makes recs on high pending writes"""
analyzer = recs.Engine(None, None)
stage = recs.Stage(
name="TPC/all/WRITE_REMOTE",
pending=10001,
active=0,
local_backpressure=0,
completed=0,
blocked=0,
all_time_blocked=0,
)
reason, rec = analyzer.analyze_stage(stage)
self.assertEqual(rec, "lower memtable_cleanup_threshold in cassandra.yaml")
self.assertEqual(reason, "pending remote writes over 10000")
stage.pending = 9999
reason, rec = analyzer.analyze_stage(stage)
self.assertIsNone(rec)
self.assertIsNone(reason)
def test_full_memtable(self):
"""verify StageAnalyzer"""
analyzer = recs.Engine(None, None)
stage = recs.Stage(
name="TPC/all/WRITE_MEMTABLE_FULL",
pending=0,
active=1,
local_backpressure=0,
completed=0,
blocked=0,
all_time_blocked=0,
)
reason, rec = analyzer.analyze_stage(stage)
self.assertEqual(rec, "lower memtable_cleanup_threshold in cassandra.yaml")
self.assertEqual(reason, "full memtable")
stage.active = 0
reason, rec = analyzer.analyze_stage(stage)
self.assertIsNone(rec)
self.assertIsNone(reason)
def test_compactions_behind(self):
"""verify compactions analysis"""
analyzer = recs.Engine(None, None)
stage = recs.Stage(
name="CompactionManger",
pending=101,
active=0,
local_backpressure=0,
completed=0,
blocked=0,
all_time_blocked=0,
)
reason, rec = analyzer.analyze_stage(stage)
self.assertEqual(rec, "raise compaction_throughput_in_mb in cassandra.yaml")
self.assertEqual(reason, "more than 100 compactions behind")
stage.pending = 0
reason, rec = analyzer.analyze_stage(stage)
self.assertIsNone(rec)
self.assertIsNone(reason)
def test_high_pending_mutations():
"""verify StageAnalyzer makes recs on high pending local writes"""
analyzer = recs.Engine()
stage = recs.Stage(
name="MutationStage",
pending=10001,
active=0,
local_backpressure=0,
completed=0,
blocked=0,
all_time_blocked=0,
)
reason, rec = analyzer.analyze_stage(stage)
assert rec == "lower memtable_cleanup_threshold in cassandra.yaml"
assert reason == "mutations pending over 10000"
stage.pending = 9999
reason, rec = analyzer.analyze_stage(stage)
assert not rec
assert not reason
def test_memtable_flush_writer_blocked():
"""verify flush writer blocked"""
analyzer = recs.Engine()
stage = recs.Stage(
name="MemtableFlushWriter",
pending=0,
active=0,
local_backpressure=0,
completed=0,
blocked=1,
all_time_blocked=0,
)
reason, rec = analyzer.analyze_stage(stage)
assert rec == "lower memtable_cleanup_threshold in cassandra.yaml"
assert reason == "memtable flush writers blocked greater than zero"
stage.blocked = 0
reason, rec = analyzer.analyze_stage(stage)
assert not rec
assert not reason
def test_memtable_flush_writer_pending():
"""verify flush writer pending"""
analyzer = recs.Engine()
stage = recs.Stage(
name="MemtableFlushWriter",
pending=6,
active=0,
local_backpressure=0,
completed=0,
blocked=0,
all_time_blocked=0,
)
reason, rec = analyzer.analyze_stage(stage)
assert rec == "raise memtable_flush_writers in cassandra.yaml"
assert reason == "memtable flush writers pending over 5"
stage.pending = 0
reason, rec = analyzer.analyze_stage(stage)
assert not rec
assert not reason
def test_compactions_behind():
"""verify compactions analysis"""
analyzer = recs.Engine()
stage = recs.Stage(
name="CompactionManger",
pending=101,
active=0,
local_backpressure=0,
completed=0,
blocked=0,
all_time_blocked=0,
)
reason, rec = analyzer.analyze_stage(stage)
assert rec == "raise compaction_throughput_in_mb in cassandra.yaml"
assert reason == "more than 100 compactions behind"
stage.pending = 0
reason, rec = analyzer.analyze_stage(stage)
assert not rec
assert not reason
def test_full_memtable_completed():
"""verify full memtable historically completed"""
analyzer = recs.Engine()
stage = recs.Stage(
name="TPC/all/WRITE_MEMTABLE_FULL",
pending=0,
active=0,
local_backpressure=0,
completed=1,
blocked=0,
all_time_blocked=0,
)
reason, rec = analyzer.analyze_stage(stage)
assert rec == "lower memtable_cleanup_threshold in cassandra.yaml"
assert reason == "full memtable stages previously completed is too high"
stage.completed = 0
reason, rec = analyzer.analyze_stage(stage)
assert not rec
assert not reason
def test_ntr_all_time_blocked():
"""verify ntr blocked"""
analyzer = recs.Engine()
stage = recs.Stage(
name="Native-Transport-Requests",
pending=0,
active=0,
local_backpressure=0,
completed=0,
blocked=0,
all_time_blocked=101,
)
reason, rec = analyzer.analyze_stage(stage)
assert rec == "raise or set -Dcassandra.max_queued_native_transport_requests= " + \
"(valid range is 1024-8192)"
assert reason == "more than 100 blocked NTR all time"
stage.all_time_blocked = 0
reason, rec = analyzer.analyze_stage(stage)
assert not rec
assert not reason
def test_tpc_backpressure():
"""verify StageAnalyzer makes recs on any backpressure"""
analyzer = recs.Engine()
stage = recs.Stage(
name="TPC/2",
pending=1,
active=0,
local_backpressure=1,
completed=0,
blocked=0,
all_time_blocked=0,
)
reason, rec = analyzer.analyze_stage(stage)
assert rec == "raise or set tpc_concurrent_requests_limit in " + \
"cassandra.yaml (default is 128), if CPU is underutilized."
assert reason == "local backpressure is present"
stage.pending = 0
stage.local_backpressure = 0
reason, rec = analyzer.analyze_stage(stage)
assert not rec
assert not reason
def test_ntr_blocked(self):
"""verify ntr blocked"""
analyzer = recs.Engine()
stage = recs.Stage(
name="Native-Transport-Requests",
pending=0,
active=0,
local_backpressure=0,
completed=0,
blocked=11,
all_time_blocked=0,
)
reason, rec = analyzer.analyze_stage(stage)
self.assertEqual(
rec,
"raise or set -Dcassandra.max_queued_native_transport_requests= " +
"(valid range is 1024-8192)",
)
self.assertEqual(reason, "blocked NTR over 10")
stage.blocked = 0
reason, rec = analyzer.analyze_stage(stage)
self.assertIsNone(rec)
self.assertIsNone(reason)
