def test_ssl_cert():
# Disable ssl warning for self signed cert/no verify
urllib3.disable_warnings()
run_ssl_server()
c = SparkCheck('spark', {}, [SSL_CERT_CONFIG])
c.check(SSL_CERT_CONFIG)
def test_standalone(aggregator, instance_standalone):
c = SparkCheck('spark', {}, [instance_standalone])
c.check(instance_standalone)
# Check the running job metrics
for metric in SPARK_JOB_RUNNING_METRIC_VALUES:
aggregator.assert_metric(metric)
# Check the succeeded job metrics
for metric in SPARK_JOB_SUCCEEDED_METRIC_VALUES:
aggregator.assert_metric(metric)
# Check the running stage metrics
for metric in SPARK_STAGE_RUNNING_METRIC_VALUES:
aggregator.assert_metric(metric)
# Check the complete stage metrics
for metric in SPARK_STAGE_COMPLETE_METRIC_VALUES:
aggregator.assert_metric(metric)
# Check the driver metrics
for metric in SPARK_DRIVER_METRIC_VALUES:
aggregator.assert_metric(metric)
# Check the streaming statistics metrics and remove 2 we're not getting for now
streaming_statistics = set(SPARK_STREAMING_STATISTICS_METRIC_VALUES) - {
'spark.streaming.statistics.avg_processing_time',
'spark.streaming.statistics.avg_total_delay',
}
for metric in streaming_statistics:
aggregator.assert_metric(metric)
def test_ssl_no_verify():
# Disable ssl warning for self signed cert/no verify
requests.packages.urllib3.disable_warnings()
run_ssl_server()
c = SparkCheck('spark', None, {}, [SSL_NO_VERIFY_CONFIG])
c.check(SSL_NO_VERIFY_CONFIG)
def test_yarn(aggregator):
with mock.patch('requests.get', yarn_requests_get_mock):
c = SparkCheck('spark', None, {}, [YARN_CONFIG])
c.check(YARN_CONFIG)
# Check the running job metrics
for metric, value in SPARK_JOB_RUNNING_METRIC_VALUES.iteritems():
aggregator.assert_metric(
metric,
tags=SPARK_JOB_RUNNING_METRIC_TAGS, value=value)
# Check the succeeded job metrics
for metric, value in SPARK_JOB_SUCCEEDED_METRIC_VALUES.iteritems():
aggregator.assert_metric(
metric,
value=value,
tags=SPARK_JOB_SUCCEEDED_METRIC_TAGS)
# Check the running stage metrics
for metric, value in SPARK_STAGE_RUNNING_METRIC_VALUES.iteritems():
aggregator.assert_metric(
metric,
value=value,
tags=SPARK_STAGE_RUNNING_METRIC_TAGS)
# Check the complete stage metrics
for metric, value in SPARK_STAGE_COMPLETE_METRIC_VALUES.iteritems():
aggregator.assert_metric(
metric,
value=value,
tags=SPARK_STAGE_COMPLETE_METRIC_TAGS)
# Check the driver metrics
for metric, value in SPARK_DRIVER_METRIC_VALUES.iteritems():
aggregator.assert_metric(
metric,
value=value,
tags=SPARK_METRIC_TAGS)
# Check the executor metrics
for metric, value in SPARK_EXECUTOR_METRIC_VALUES.iteritems():
aggregator.assert_metric(
metric,
value=value,
tags=SPARK_METRIC_TAGS)
# Check the RDD metrics
for metric, value in SPARK_RDD_METRIC_VALUES.iteritems():
aggregator.assert_metric(
metric,
value=value,
tags=SPARK_METRIC_TAGS)
for sc in aggregator.service_checks(YARN_SERVICE_CHECK):
assert sc.status == SparkCheck.OK
assert sc.tags == ['url:http://localhost:8088']
for sc in aggregator.service_checks(SPARK_SERVICE_CHECK):
assert sc.status == SparkCheck.OK
assert sc.tags == ['url:http://localhost:8088']
def test_driver_unit(aggregator):
with mock.patch('requests.get', driver_requests_get_mock):
c = SparkCheck('spark', {}, [DRIVER_CONFIG])
c.check(DRIVER_CONFIG)
# Check the running job metrics
for metric, value in iteritems(SPARK_JOB_RUNNING_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_JOB_RUNNING_METRIC_TAGS + CUSTOM_TAGS)
# Check the succeeded job metrics
for metric, value in iteritems(SPARK_JOB_SUCCEEDED_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_JOB_SUCCEEDED_METRIC_TAGS + CUSTOM_TAGS)
# Check the running stage metrics
for metric, value in iteritems(SPARK_STAGE_RUNNING_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_STAGE_RUNNING_METRIC_TAGS + CUSTOM_TAGS)
# Check the complete stage metrics
for metric, value in iteritems(SPARK_STAGE_COMPLETE_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_STAGE_COMPLETE_METRIC_TAGS + CUSTOM_TAGS)
# Check the driver metrics
for metric, value in iteritems(SPARK_DRIVER_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_METRIC_TAGS + CUSTOM_TAGS)
# Check the executor level metrics
for metric, value in iteritems(SPARK_EXECUTOR_LEVEL_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_EXECUTOR_LEVEL_METRIC_TAGS + CUSTOM_TAGS)
# Check the summary executor metrics
for metric, value in iteritems(SPARK_EXECUTOR_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_METRIC_TAGS + CUSTOM_TAGS)
# Check the RDD metrics
for metric, value in iteritems(SPARK_RDD_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_METRIC_TAGS + CUSTOM_TAGS)
# Check the streaming statistics metrics
for metric, value in iteritems(SPARK_STREAMING_STATISTICS_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_METRIC_TAGS + CUSTOM_TAGS)
# Check the service tests
for sc in aggregator.service_checks(SPARK_DRIVER_SERVICE_CHECK):
assert sc.status == SparkCheck.OK
tags = ['url:http://localhost:4040', 'cluster_name:SparkCluster'] + CUSTOM_TAGS
tags.sort()
sc.tags.sort()
assert sc.tags == tags
for sc in aggregator.service_checks(SPARK_SERVICE_CHECK):
assert sc.status == SparkCheck.OK
tags = ['url:http://localhost:4040', 'cluster_name:SparkCluster'] + CUSTOM_TAGS
tags.sort()
sc.tags.sort()
assert sc.tags == tags
# Assert coverage for this check on this instance
aggregator.assert_all_metrics_covered()
def test_do_not_crash_on_single_app_failure():
running_apps = {'foo': ('bar', 'http://foo.bar/'), 'foo2': ('bar', 'http://foo.bar/')}
results = []
rest_requests_to_json = mock.MagicMock(side_effect=[RequestException, results])
c = SparkCheck('spark', {}, [INSTANCE_STANDALONE])
with mock.patch.object(c, '_rest_request_to_json', rest_requests_to_json), mock.patch.object(c, '_collect_version'):
c._get_spark_app_ids(running_apps, [])
assert rest_requests_to_json.call_count == 2
def test_do_not_crash_on_version_collection_failure():
running_apps = {'foo': ('bar', 'http://foo.bar/'), 'foo2': ('bar', 'http://foo.bar/')}
rest_requests_to_json = mock.MagicMock(side_effect=[RequestException, []])
c = SparkCheck('spark', {}, [INSTANCE_STANDALONE])
with mock.patch.object(c, '_rest_request_to_json', rest_requests_to_json):
# ensure no exception is raised by calling collect_version
assert not c._collect_version(running_apps, [])
def test_mesos_filter(aggregator):
with mock.patch('requests.get', mesos_requests_get_mock):
c = SparkCheck('spark', {}, [MESOS_FILTERED_CONFIG])
c.check(MESOS_FILTERED_CONFIG)
for sc in aggregator.service_checks(MESOS_SERVICE_CHECK):
assert sc.status == SparkCheck.OK
assert sc.tags == ['url:http://localhost:5050', 'cluster_name:SparkCluster']
assert aggregator.metrics_asserted_pct == 100.0
def test_standalone_unit_with_proxy_warning_page(aggregator):
c = SparkCheck('spark', {}, [STANDALONE_CONFIG])
with mock.patch('requests.get', proxy_with_warning_page_mock):
c.check(STANDALONE_CONFIG)
# Check the running job metrics
for metric, value in iteritems(SPARK_JOB_RUNNING_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_JOB_RUNNING_METRIC_TAGS)
# Check the running job metrics
for metric, value in iteritems(SPARK_JOB_RUNNING_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_JOB_RUNNING_METRIC_TAGS)
# Check the succeeded job metrics
for metric, value in iteritems(SPARK_JOB_SUCCEEDED_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_JOB_SUCCEEDED_METRIC_TAGS)
# Check the running stage metrics
for metric, value in iteritems(SPARK_STAGE_RUNNING_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_STAGE_RUNNING_METRIC_TAGS)
# Check the complete stage metrics
for metric, value in iteritems(SPARK_STAGE_COMPLETE_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_STAGE_COMPLETE_METRIC_TAGS)
# Check the driver metrics
for metric, value in iteritems(SPARK_DRIVER_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_METRIC_TAGS)
# Check the executor level metrics
for metric, value in iteritems(SPARK_EXECUTOR_LEVEL_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_EXECUTOR_LEVEL_METRIC_TAGS)
# Check the summary executor metrics
for metric, value in iteritems(SPARK_EXECUTOR_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_METRIC_TAGS)
# Check the RDD metrics
for metric, value in iteritems(SPARK_RDD_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_METRIC_TAGS)
# Check the streaming statistics metrics
for metric, value in iteritems(SPARK_STREAMING_STATISTICS_METRIC_VALUES):
aggregator.assert_metric(metric, value=value, tags=SPARK_METRIC_TAGS)
# Check the service tests
for sc in aggregator.service_checks(STANDALONE_SERVICE_CHECK):
assert sc.status == SparkCheck.OK
assert sc.tags == ['url:http://localhost:8080', 'cluster_name:SparkCluster']
for sc in aggregator.service_checks(SPARK_SERVICE_CHECK):
assert sc.status == SparkCheck.OK
assert sc.tags == ['url:http://localhost:4040', 'cluster_name:SparkCluster']
# Assert coverage for this check on this instance
aggregator.assert_all_metrics_covered()
def test_integration(aggregator, instance, service_check, cluster_name,
spark_url):
c = SparkCheck('spark', {}, [instance])
c.check(instance)
# Check the running job metrics
for metric in SPARK_JOB_RUNNING_METRIC_VALUES:
aggregator.assert_metric(metric)
# Check the succeeded job metrics
for metric in SPARK_JOB_SUCCEEDED_METRIC_VALUES:
aggregator.assert_metric(metric)
# Check the running stage metrics
for metric in SPARK_STAGE_RUNNING_METRIC_VALUES:
aggregator.assert_metric(metric)
# Check the complete stage metrics
for metric in SPARK_STAGE_COMPLETE_METRIC_VALUES:
aggregator.assert_metric(metric)
# Check the driver metrics
for metric in SPARK_DRIVER_METRIC_VALUES:
aggregator.assert_metric(metric)
# Check the streaming statistics metrics and remove 2 we're not getting for now
streaming_statistics = set(SPARK_STREAMING_STATISTICS_METRIC_VALUES) - {
'spark.streaming.statistics.avg_processing_time',
'spark.streaming.statistics.avg_total_delay',
}
for metric in streaming_statistics:
aggregator.assert_metric(metric)
# TODO: Further investigate why we aren't getting these, it may be Docker networking
#
# # Check the executor metrics
# for metric in SPARK_EXECUTOR_METRIC_VALUES:
# aggregator.assert_metric(metric)
#
# # Check the RDD metrics
# for metric in SPARK_RDD_METRIC_VALUES:
# aggregator.assert_metric(metric)
aggregator.assert_service_check(
service_check,
status=SparkCheck.OK,
tags=[
'cluster_name:{}'.format(cluster_name), 'url:{}'.format(spark_url)
],
)
def test_auth_yarn(aggregator):
with mock.patch('requests.get', yarn_requests_auth_mock):
c = SparkCheck('spark', {}, [YARN_AUTH_CONFIG])
c.check(YARN_AUTH_CONFIG)
tags = ['url:http://localhost:8088', 'cluster_name:SparkCluster'] + CUSTOM_TAGS
tags.sort()
for sc in aggregator.service_checks(YARN_SERVICE_CHECK):
assert sc.status == SparkCheck.OK
sc.tags.sort()
assert sc.tags == tags
for sc in aggregator.service_checks(SPARK_SERVICE_CHECK):
assert sc.status == SparkCheck.OK
sc.tags.sort()
assert sc.tags == tags
def test_metadata(aggregator, datadog_agent):
with mock.patch('requests.get', standalone_requests_pre20_get_mock):
c = SparkCheck(CHECK_NAME, {}, [STANDALONE_CONFIG_PRE_20])
c.check_id = "test:123"
c.check(STANDALONE_CONFIG_PRE_20)
c._collect_version(SPARK_APP_URL, None)
raw_version = "2.4.0"
major, minor, patch = raw_version.split(".")
version_metadata = {
'version.major': major,
'version.minor': minor,
'version.patch': patch,
'version.raw': raw_version,
}
datadog_agent.assert_metadata('test:123', version_metadata)
def test_ssl():
run_ssl_server()
c = SparkCheck('spark', {}, [SSL_CONFIG])
with pytest.raises(requests.exceptions.SSLError):
c.check(SSL_CONFIG)
def test_standalone_pre20(aggregator):
with mock.patch('requests.get', standalone_requests_pre20_get_mock):
c = SparkCheck('spark', None, {}, [STANDALONE_CONFIG_PRE_20])
c.check(STANDALONE_CONFIG_PRE_20)
# Check the running job metrics
for metric, value in SPARK_JOB_RUNNING_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_JOB_RUNNING_METRIC_TAGS)
# Check the running job metrics
for metric, value in SPARK_JOB_RUNNING_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_JOB_RUNNING_METRIC_TAGS)
# Check the succeeded job metrics
for metric, value in SPARK_JOB_SUCCEEDED_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_JOB_SUCCEEDED_METRIC_TAGS)
# Check the running stage metrics
for metric, value in SPARK_STAGE_RUNNING_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_STAGE_RUNNING_METRIC_TAGS)
# Check the complete stage metrics
for metric, value in SPARK_STAGE_COMPLETE_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_STAGE_COMPLETE_METRIC_TAGS)
# Check the driver metrics
for metric, value in SPARK_DRIVER_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_METRIC_TAGS)
# Check the executor metrics
for metric, value in SPARK_EXECUTOR_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_METRIC_TAGS)
# Check the RDD metrics
for metric, value in SPARK_RDD_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_METRIC_TAGS)
# Check the service tests
for sc in aggregator.service_checks(STANDALONE_SERVICE_CHECK):
assert sc.status == SparkCheck.OK
assert sc.tags == [
'url:http://localhost:8080', 'cluster_name:SparkCluster'
]
for sc in aggregator.service_checks(SPARK_SERVICE_CHECK):
assert sc.status == SparkCheck.OK
assert sc.tags == [
'url:http://localhost:4040', 'cluster_name:SparkCluster'
]
def test_mesos(aggregator):
with mock.patch('requests.get', mesos_requests_get_mock):
c = SparkCheck('spark', None, {}, [MESOS_CONFIG])
c.check(MESOS_CONFIG)
# Check the running job metrics
for metric, value in SPARK_JOB_RUNNING_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_JOB_RUNNING_METRIC_TAGS +
CUSTOM_TAGS)
# Check the succeeded job metrics
for metric, value in SPARK_JOB_SUCCEEDED_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_JOB_SUCCEEDED_METRIC_TAGS +
CUSTOM_TAGS)
# Check the running stage metrics
for metric, value in SPARK_STAGE_RUNNING_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_STAGE_RUNNING_METRIC_TAGS +
CUSTOM_TAGS)
# Check the complete stage metrics
for metric, value in SPARK_STAGE_COMPLETE_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_STAGE_COMPLETE_METRIC_TAGS +
CUSTOM_TAGS)
# Check the driver metrics
for metric, value in SPARK_DRIVER_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_METRIC_TAGS + CUSTOM_TAGS)
# Check the executor metrics
for metric, value in SPARK_EXECUTOR_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_METRIC_TAGS + CUSTOM_TAGS)
# Check the RDD metrics
for metric, value in SPARK_RDD_METRIC_VALUES.iteritems():
aggregator.assert_metric(metric,
value=value,
tags=SPARK_METRIC_TAGS + CUSTOM_TAGS)
# Check the service tests
for sc in aggregator.service_checks(MESOS_SERVICE_CHECK):
assert sc.status == SparkCheck.OK
tags = ['url:http://localhost:5050', 'cluster_name:SparkCluster'
] + CUSTOM_TAGS
tags.sort()
sc.tags.sort()
assert sc.tags == tags
for sc in aggregator.service_checks(SPARK_SERVICE_CHECK):
assert sc.status == SparkCheck.OK
tags = ['url:http://localhost:4040', 'cluster_name:SparkCluster'
] + CUSTOM_TAGS
tags.sort()
sc.tags.sort()
assert sc.tags == tags
assert aggregator.metrics_asserted_pct == 100.0