def test_check(aggregator, mocked_request):
instance = YARN_CONFIG['instances'][0]
# Instantiate YarnCheck
yarn = YarnCheck('yarn', {}, [instance])
# Run the check once
yarn.check(instance)
aggregator.assert_service_check(
SERVICE_CHECK_NAME,
status=YarnCheck.OK,
tags=YARN_CLUSTER_METRICS_TAGS + CUSTOM_TAGS + ['url:{}'.format(RM_ADDRESS)],
)
aggregator.assert_service_check(
APPLICATION_STATUS_SERVICE_CHECK,
status=YarnCheck.OK,
tags=['app_queue:default', 'app_name:word count', 'optional:tag1', 'cluster_name:SparkCluster'],
)
aggregator.assert_service_check(
APPLICATION_STATUS_SERVICE_CHECK,
status=YarnCheck.CRITICAL,
tags=['app_queue:default', 'app_name:dead app', 'optional:tag1', 'cluster_name:SparkCluster'],
)
aggregator.assert_service_check(
APPLICATION_STATUS_SERVICE_CHECK,
status=YarnCheck.OK,
tags=['app_queue:default', 'app_name:new app', 'optional:tag1', 'cluster_name:SparkCluster'],
)
# Check the YARN Cluster Metrics
for metric, value in iteritems(YARN_CLUSTER_METRICS_VALUES):
aggregator.assert_metric(metric, value=value, tags=YARN_CLUSTER_METRICS_TAGS + CUSTOM_TAGS, count=1)
# Check the YARN App Metrics
for metric, value in iteritems(YARN_APP_METRICS_VALUES):
aggregator.assert_metric(metric, value=value, tags=YARN_APP_METRICS_TAGS + CUSTOM_TAGS, count=1)
for metric, value in iteritems(DEPRECATED_YARN_APP_METRICS_VALUES):
aggregator.assert_metric(metric, value=value, tags=YARN_APP_METRICS_TAGS + CUSTOM_TAGS, count=1)
# Check the YARN Node Metrics
for metric, value in iteritems(YARN_NODE_METRICS_VALUES):
aggregator.assert_metric(metric, value=value, tags=YARN_NODE_METRICS_TAGS + CUSTOM_TAGS, count=1)
# Check the YARN Root Queue Metrics
for metric, value in iteritems(YARN_ROOT_QUEUE_METRICS_VALUES):
aggregator.assert_metric(metric, value=value, tags=YARN_ROOT_QUEUE_METRICS_TAGS + CUSTOM_TAGS, count=1)
# Check the YARN Custom Queue Metrics
for metric, value in iteritems(YARN_QUEUE_METRICS_VALUES):
aggregator.assert_metric(metric, value=value, tags=YARN_QUEUE_METRICS_TAGS + CUSTOM_TAGS, count=1)
# Check the YARN Queue Metrics from excluded queues are absent
for metric, _ in YARN_QUEUE_METRICS.values():
aggregator.assert_metric(metric, tags=YARN_QUEUE_NOFOLLOW_METRICS_TAGS + CUSTOM_TAGS, count=0)
aggregator.assert_all_metrics_covered()
def test_check_mapping(aggregator, mocked_request):
instance = YARN_CONFIG_STATUS_MAPPING['instances'][0]
# Instantiate YarnCheck
yarn = YarnCheck('yarn', {}, [instance])
# Run the check once
yarn.check(instance)
aggregator.assert_service_check(
SERVICE_CHECK_NAME,
status=YarnCheck.OK,
tags=YARN_CLUSTER_METRICS_TAGS + CUSTOM_TAGS + ['url:{}'.format(RM_ADDRESS)],
)
aggregator.assert_service_check(
APPLICATION_STATUS_SERVICE_CHECK,
status=YarnCheck.OK,
tags=['app_queue:default', 'app_name:word count', 'optional:tag1', 'cluster_name:SparkCluster'],
)
aggregator.assert_service_check(
APPLICATION_STATUS_SERVICE_CHECK,
status=YarnCheck.WARNING,
tags=['app_queue:default', 'app_name:dead app', 'optional:tag1', 'cluster_name:SparkCluster'],
)
aggregator.assert_service_check(
APPLICATION_STATUS_SERVICE_CHECK,
status=YarnCheck.OK,
tags=['app_queue:default', 'app_name:new app', 'optional:tag1', 'cluster_name:SparkCluster'],
)
def test_custom_mapping(aggregator, mocked_request):
instance = copy.deepcopy(YARN_CONFIG['instances'][0])
instance['application_status_mapping'] = {'KILLED': 'WARNING', 'RUNNING': 'OK'}
yarn = YarnCheck('yarn', {}, [instance])
# Run the check once
yarn.check(instance)
aggregator.assert_service_check(
APPLICATION_STATUS_SERVICE_CHECK,
status=YarnCheck.OK,
tags=['app_queue:default', 'app_name:word count', 'optional:tag1', 'cluster_name:SparkCluster'],
)
aggregator.assert_service_check(
APPLICATION_STATUS_SERVICE_CHECK,
status=YarnCheck.WARNING,
tags=['app_queue:default', 'app_name:dead app', 'optional:tag1', 'cluster_name:SparkCluster'],
)
aggregator.assert_service_check(
APPLICATION_STATUS_SERVICE_CHECK,
status=YarnCheck.UNKNOWN,
tags=['app_queue:default', 'app_name:new app', 'optional:tag1', 'cluster_name:SparkCluster'],
)
def test_disable_legacy_cluster_tag(aggregator, mocked_request):
instance = YARN_CONFIG_SPLIT_APPLICATION_TAGS['instances'][0]
instance['disable_legacy_cluster_tag'] = True
# Instantiate YarnCheck
yarn = YarnCheck('yarn', {}, [instance])
# Run the check once
yarn.check(instance)
# Check that the YARN application tags have been split for properly formatted tags without cluster_name tag
expected_tags = CUSTOM_TAGS
expected_tags.append(YARN_CLUSTER_TAG)
aggregator.assert_service_check(
APPLICATION_STATUS_SERVICE_CHECK,
status=YarnCheck.OK,
tags=[
'app_queue:default',
'app_name:word count',
'app_key1:value1',
'app_key2:value2',
] + expected_tags,
)
# And check that the YARN application tags have not been split for other tags
aggregator.assert_service_check(
APPLICATION_STATUS_SERVICE_CHECK,
status=YarnCheck.WARNING,
tags=[
'app_queue:default',
'app_name:dead app',
'app_tags:tag1,tag2',
] + expected_tags,
)
def test_check_splits_yarn_application_tags(aggregator, mocked_request):
instance = YARN_CONFIG_SPLIT_APPLICATION_TAGS['instances'][0]
# Instantiate YarnCheck
yarn = YarnCheck('yarn', {}, [instance])
# Run the check once
yarn.check(instance)
# Check that the YARN application tags have been split for properly formatted tags
aggregator.assert_service_check(
APPLICATION_STATUS_SERVICE_CHECK,
status=YarnCheck.OK,
tags=[
'app_queue:default',
'app_name:word count',
'app_key1:value1',
'app_key2:value2',
'optional:tag1',
'cluster_name:SparkCluster',
],
)
# And check that the YARN application tags have not been split for other tags
aggregator.assert_service_check(
APPLICATION_STATUS_SERVICE_CHECK,
status=YarnCheck.WARNING,
tags=[
'app_queue:default',
'app_name:dead app',
'app_tags:tag1,tag2',
'optional:tag1',
'cluster_name:SparkCluster',
],
)
def test_split_application_tag(tags, expected_tags):
instance = YARN_CONFIG['instances'][0]
# Instantiate YarnCheck
yarn = YarnCheck('yarn', {}, [instance])
split_tags = yarn._split_yarn_application_tags(tags, "job_tag")
assert split_tags == expected_tags
def test_check(aggregator, mocked_request):
# Instantiate YarnCheck
yarn = YarnCheck('yarn', {}, {})
# Run the check once
yarn.check(YARN_CONFIG['instances'][0])
aggregator.assert_service_check(
SERVICE_CHECK_NAME,
status=YarnCheck.OK,
tags=YARN_CLUSTER_METRICS_TAGS + CUSTOM_TAGS +
['url:{}'.format(RM_ADDRESS)],
)
# Check the YARN Cluster Metrics
for metric, value in iteritems(YARN_CLUSTER_METRICS_VALUES):
aggregator.assert_metric(metric,
value=value,
tags=YARN_CLUSTER_METRICS_TAGS + CUSTOM_TAGS,
count=1)
# Check the YARN App Metrics
for metric, value in iteritems(YARN_APP_METRICS_VALUES):
aggregator.assert_metric(metric,
value=value,
tags=YARN_APP_METRICS_TAGS + CUSTOM_TAGS,
count=1)
# Check the YARN Node Metrics
for metric, value in iteritems(YARN_NODE_METRICS_VALUES):
aggregator.assert_metric(metric,
value=value,
tags=YARN_NODE_METRICS_TAGS + CUSTOM_TAGS,
count=1)
# Check the YARN Root Queue Metrics
for metric, value in iteritems(YARN_ROOT_QUEUE_METRICS_VALUES):
aggregator.assert_metric(metric,
value=value,
tags=YARN_ROOT_QUEUE_METRICS_TAGS +
CUSTOM_TAGS,
count=1)
# Check the YARN Custom Queue Metrics
for metric, value in iteritems(YARN_QUEUE_METRICS_VALUES):
aggregator.assert_metric(metric,
value=value,
tags=YARN_QUEUE_METRICS_TAGS + CUSTOM_TAGS,
count=1)
# Check the YARN Queue Metrics from excluded queues are absent
for metric, _ in YARN_QUEUE_METRICS.values():
aggregator.assert_metric(metric,
tags=YARN_QUEUE_NOFOLLOW_METRICS_TAGS +
CUSTOM_TAGS,
count=0)
aggregator.assert_all_metrics_covered()
def test_ssl_verification(aggregator, mocked_bad_cert_request):
instance = YARN_SSL_VERIFY_TRUE_CONFIG['instances'][0]
# Instantiate YarnCheck
yarn = YarnCheck('yarn', {}, [instance])
# Run the check on a config with a badly configured SSL certificate
try:
yarn.check(instance)
except SSLError:
aggregator.assert_service_check(
SERVICE_CHECK_NAME,
status=YarnCheck.CRITICAL,
tags=YARN_CLUSTER_METRICS_TAGS + CUSTOM_TAGS + ['url:{}'.format(RM_ADDRESS)],
count=1,
)
pass
else:
raise AssertionError('Should have thrown an SSLError due to a badly configured certificate')
# Run the check on the same configuration, but with verify=False. We shouldn't get an exception.
instance = YARN_SSL_VERIFY_FALSE_CONFIG['instances'][0]
yarn = YarnCheck('yarn', {}, [instance])
yarn.check(instance)
aggregator.assert_service_check(
SERVICE_CHECK_NAME,
status=YarnCheck.OK,
tags=YARN_CLUSTER_METRICS_TAGS + CUSTOM_TAGS + ['url:{}'.format(RM_ADDRESS)],
count=4,
)
def test_auth(aggregator, mocked_auth_request):
# Instantiate YarnCheck
yarn = YarnCheck('yarn', {}, {})
# Run the check once
yarn.check(YARN_AUTH_CONFIG['instances'][0])
# Make sure check is working
aggregator.assert_service_check(
SERVICE_CHECK_NAME,
status=YarnCheck.OK,
tags=YARN_CLUSTER_METRICS_TAGS + CUSTOM_TAGS + ['url:{}'.format(RM_ADDRESS)],
count=4,
)
def test_check_excludes_app_metrics(aggregator, mocked_request):
# Instantiate YarnCheck
yarn = YarnCheck('yarn', {}, {})
# Run the check once
yarn.check(YARN_CONFIG_EXCLUDING_APP['instances'][0])
# Check that the YARN App metrics is empty
for metric, type in YARN_APP_METRICS.values():
aggregator.assert_metric(metric, count=0)
# Check that our service is up
aggregator.assert_service_check(
SERVICE_CHECK_NAME,
status=YarnCheck.OK,
tags=YARN_CLUSTER_METRICS_TAGS + CUSTOM_TAGS + ['url:{}'.format(RM_ADDRESS)],
count=3,
)
def test_metadata(aggregator, instance, datadog_agent):
check = YarnCheck("yarn", {}, [instance])
check.check_id = "test:123"
check.check(instance)
raw_version = os.getenv("YARN_VERSION")
major, minor, patch = raw_version.split(".")
version_metadata = {
"version.scheme": "semver",
"version.major": major,
"version.minor": minor,
"version.patch": patch,
"version.raw": raw_version,
}
datadog_agent.assert_metadata("test:123", version_metadata)
def check():
return lambda instance: YarnCheck('yarn', {}, [instance])
def check():
return YarnCheck('yarn', {}, {})