def test_get_storm_topology_summary():
with mock.patch('datadog_checks.storm.StormCheck.get_request_json',
return_value=TEST_STORM_TOPOLOGY_SUMMARY):
check = StormCheck(CHECK_NAME, {}, {})
check.update_from_config(STORM_CHECK_CONFIG)
result = check.get_storm_topology_summary()
assert result == TEST_STORM_TOPOLOGY_SUMMARY
def test_get_storm_topology_metrics():
with mock.patch('datadog_checks.storm.StormCheck.get_request_json',
return_value=TEST_STORM_TOPOLOGY_METRICS_RESP):
check = StormCheck(CHECK_NAME, {}, {})
check.update_from_config(STORM_CHECK_CONFIG)
result = check.get_topology_metrics('my_topology-1-1489183263')
assert result == TEST_STORM_TOPOLOGY_METRICS_RESP
def test_load_from_config():
check = StormCheck(CHECK_NAME, {}, {})
check.update_from_config(STORM_CHECK_CONFIG)
assert check.nimbus_server == 'http://localhost:8080'
assert check.environment_name == 'test'
assert check.additional_tags == []
assert check.excluded_topologies == []
assert check.intervals == [60]
def test_process_topology_stats():
check = StormCheck(CHECK_NAME, {}, {})
check.update_from_config(STORM_CHECK_CONFIG)
results = defaultdict(list)
def report_histogram(metric, value, tags, additional_tags):
results[metric].append((value, tags, additional_tags))
check.report_histogram = report_histogram
check.process_topology_stats(TEST_STORM_TOPOLOGY_RESP, interval=60)
assert len(results) == 47
# Check Topology Stats
assert results['storm.topologyStats.last_60.emitted'][0][0] == 307606
assert results['storm.topologyStats.last_60.transferred'][0][0] == 307606
assert results['storm.topologyStats.last_60.acked'][0][0] == 104673
assert results['storm.topologyStats.last_60.failed'][0][0] == 0
assert results['storm.topologyStats.last_60.completeLatency'][0][
0] == 285.950
assert results['storm.topologyStats.last_60.uptimeSeconds'][0][
0] == 1525788
assert results['storm.topologyStats.last_60.executorsTotal'][0][0] == 33
assert results['storm.topologyStats.last_60.numBolts'][0][0] == 6
assert results['storm.topologyStats.last_60.replicationCount'][0][0] == 1
assert results['storm.topologyStats.last_60.tasksTotal'][0][0] == 33
assert results['storm.topologyStats.last_60.numSpouts'][0][0] == 1
assert results['storm.topologyStats.last_60.workersTotal'][0][0] == 6
# Check Bolt Stats
assert results['storm.bolt.last_60.tasks'][0][0] == 3
assert 'bolt:Bolt1' in results['storm.bolt.last_60.tasks'][0][1]
assert results['storm.bolt.last_60.executeLatency'][0][0] == 0.001
assert results['storm.bolt.last_60.processLatency'][0][0] == 201.474
assert results['storm.bolt.last_60.capacity'][0][0] == 0.000
assert results['storm.bolt.last_60.failed'][0][0] == 0
assert results['storm.bolt.last_60.emitted'][0][0] == 101309
assert results['storm.bolt.last_60.acked'][0][0] == 212282
assert results['storm.bolt.last_60.transferred'][0][0] == 101309
assert results['storm.bolt.last_60.executed'][0][0] == 106311
assert results['storm.bolt.last_60.executors'][0][0] == 3
assert results['storm.bolt.last_60.errorLapsedSecs'][0][0] == 1e10
# Check Spout Stats
assert results['storm.spout.last_60.tasks'][0][0] == 8
assert 'spout:source' in results['storm.spout.last_60.tasks'][0][1]
assert results['storm.spout.last_60.completeLatency'][0][0] == 285.950
assert results['storm.spout.last_60.failed'][0][0] == 0
assert results['storm.spout.last_60.acked'][0][0] == 104673
assert results['storm.spout.last_60.transferred'][0][0] == 104673
assert results['storm.spout.last_60.emitted'][0][0] == 104673
assert results['storm.spout.last_60.executors'][0][0] == 8
assert results['storm.spout.last_60.errorLapsedSecs'][0][0] == 38737
def test_process_supervisor_stats():
check = StormCheck(CHECK_NAME, {}, {})
check.update_from_config(STORM_CHECK_CONFIG)
results = defaultdict(list)
def report_gauge(metric, value, tags, additional_tags):
results[metric].append(value)
check.report_gauge = report_gauge
check.process_supervisor_stats(TEST_STORM_SUPERVISOR_SUMMARY)
assert len(results) == 7
# Check Supervisor Stats
assert results['storm.supervisor.uptimeSeconds'][0] == 31559
assert results['storm.supervisor.slotsTotal'][0] == 10
assert results['storm.supervisor.slotsUsed'][0] == 6
assert results['storm.supervisor.totalMem'][0] == 3072
assert results['storm.supervisor.usedMem'][0] == 4992
assert results['storm.supervisor.totalCpu'][0] == 900
assert results['storm.supervisor.usedCpu'][0] == 0
def test_process_cluster_stats():
check = StormCheck(CHECK_NAME, {}, {})
check.update_from_config(STORM_CHECK_CONFIG)
results = defaultdict(list)
def report_gauge(metric, value, tags, additional_tags):
results[metric].append(value)
check.report_gauge = report_gauge
check.process_cluster_stats(TEST_STORM_CLUSTER_SUMMARY)
assert len(results) == 13
# Check Cluster Stats
assert results['storm.cluster.executorsTotal'][0] == 33
assert results['storm.cluster.slotsTotal'][0] == 10
assert results['storm.cluster.slotsFree'][0] == 4
assert results['storm.cluster.topologies'][0] == 1
assert results['storm.cluster.supervisors'][0] == 1
assert results['storm.cluster.tasksTotal'][0] == 33
assert results['storm.cluster.slotsUsed'][0] == 6
def test_process_nimbus_stats():
check = StormCheck(CHECK_NAME, {}, {})
check.update_from_config(STORM_CHECK_CONFIG)
results = defaultdict(list)
def report_gauge(metric, value, tags, additional_tags):
results[metric].append(value)
check.report_gauge = report_gauge
check.process_nimbus_stats(TEST_STORM_NIMBUSES_SUMMARY)
assert len(results) == 5
# Check Leader Stats
assert results['storm.nimbus.upTimeSeconds'][0] == 0
assert results['storm.nimbus.upTimeSeconds'][1] == 25842
# Check General Stats
assert results['storm.nimbus.numLeaders'][0] == 1
assert results['storm.nimbus.numFollowers'][0] == 0
assert results['storm.nimbus.numOffline'][0] == 1
assert results['storm.nimbus.numDead'][0] == 0
def test_process_topology_metrics():
check = StormCheck(CHECK_NAME, {}, {})
check.update_from_config(STORM_CHECK_CONFIG)
results = defaultdict(list)
def report_histogram(metric, value, tags, additional_tags):
results[metric].append((value, tags, additional_tags))
check.report_histogram = report_histogram
check.process_topology_metrics('test', TEST_STORM_TOPOLOGY_METRICS_RESP,
60)
assert len(results) == 10
# Check Bolt Stats
assert results['storm.topologyStats.metrics.bolts.last_60.emitted'][0][
0] == 120
assert 'stream:__metrics' in results[
'storm.topologyStats.metrics.bolts.last_60.emitted'][0][1]
assert results['storm.topologyStats.metrics.bolts.last_60.emitted'][1][
0] == 190748180
assert 'stream:default' in results[
'storm.topologyStats.metrics.bolts.last_60.emitted'][1][1]
assert results['storm.topologyStats.metrics.bolts.last_60.emitted'][2][
0] == 190718100
assert 'stream:__ack_ack' in results[
'storm.topologyStats.metrics.bolts.last_60.emitted'][2][1]
assert results['storm.topologyStats.metrics.bolts.last_60.emitted'][3][
0] == 20
assert 'stream:__system' in results[
'storm.topologyStats.metrics.bolts.last_60.emitted'][3][1]
assert results['storm.topologyStats.metrics.bolts.last_60.transferred'][0][
0] == 120
assert results['storm.topologyStats.metrics.bolts.last_60.acked'][0][
0] == 190733160
assert len(
results['storm.topologyStats.metrics.bolts.last_60.failed']) == 0
assert len(
results['storm.topologyStats.metrics.bolts.last_60.complete_ms_avg']
) == 0
assert results['storm.topologyStats.metrics.bolts.last_60.process_ms_avg'][
0][0] == 0.004
assert results['storm.topologyStats.metrics.bolts.last_60.executed'][0][
0] == 190733140
assert results[
'storm.topologyStats.metrics.bolts.last_60.executed_ms_avg'][0][
0] == 0.005
# Check Spout Stats
assert results['storm.topologyStats.metrics.spouts.last_60.emitted'][0][
0] == 20
assert 'stream:__metrics' in results[
'storm.topologyStats.metrics.spouts.last_60.emitted'][0][1]
assert results['storm.topologyStats.metrics.spouts.last_60.emitted'][1][
0] == 17350280
assert 'stream:default' in results[
'storm.topologyStats.metrics.spouts.last_60.emitted'][1][1]
assert results['storm.topologyStats.metrics.spouts.last_60.emitted'][2][
0] == 17328160
assert 'stream:__ack_init' in results[
'storm.topologyStats.metrics.spouts.last_60.emitted'][2][1]
assert results['storm.topologyStats.metrics.spouts.last_60.emitted'][3][
0] == 20
assert 'stream:__system' in results[
'storm.topologyStats.metrics.spouts.last_60.emitted'][3][1]
assert results['storm.topologyStats.metrics.spouts.last_60.transferred'][
0][0] == 20
assert results['storm.topologyStats.metrics.spouts.last_60.acked'][0][
0] == 17339180
assert len(
results['storm.topologyStats.metrics.spouts.last_60.failed']) == 0
assert len(
results['storm.topologyStats.metrics.spouts.last_60.process_ms_avg']
) == 0
assert len(
results['storm.topologyStats.metrics.spouts.last_60.executed_ms_avg']
) == 0
assert len(
results['storm.topologyStats.metrics.spouts.last_60.executed']) == 0
assert results[
'storm.topologyStats.metrics.spouts.last_60.complete_ms_avg'][0][
0] == 920.497