def test_filter_with_restart():
with run_agent("""
monitors:
- type: collectd/signalfx-metadata
- type: collectd/df
- type: collectd/memory
- type: collectd/uptime
metricsToExclude:
- metricNames:
- memory.*
monitorType: collectd/memory
""") as [backend, _, update_config]:
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "df_complex.free"))
assert ensure_always(
lambda: not has_datapoint_with_metric_name(backend, "memory.used"))
assert ensure_always(
lambda: not has_datapoint_with_metric_name(backend, "memory.free"))
update_config("""
monitors:
- type: collectd/signalfx-metadata
- type: collectd/df
- type: collectd/memory
- type: collectd/uptime
metricsToExclude:
- metricNames:
- memory.used
monitorType: collectd/memory
""")
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "memory.free"))
def test_negated_filter_with_monitor_type():
with run_agent("""
monitors:
- type: collectd/signalfx-metadata
- type: collectd/memory
- type: collectd/df
- type: collectd/uptime
metricsToExclude:
- metricNames:
- memory.used
- memory.free
monitorType: collectd/memory
negated: true
- metricName: uptime
""") as [backend, _, _]:
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "memory.used"))
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "memory.free"))
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "df_complex.free"))
assert ensure_always(
lambda: not has_datapoint_with_metric_name(backend, "memory.cached"
), 10)
assert ensure_always(
lambda: not has_datapoint_with_metric_name(backend, "uptime"), 5)
def test_negated_filter_with_monitor_type():
"""
Having monitorType in a filter should make that filter only apply to a
specific monitor type and not to other metrics.
"""
with run_agent("""
monitors:
- type: collectd/signalfx-metadata
- type: collectd/memory
- type: collectd/df
- type: collectd/uptime
metricsToExclude:
- metricNames:
- memory.used
- memory.free
monitorType: collectd/memory
negated: true
- metricName: uptime
""") as [backend, _, _]:
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "memory.used"))
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "memory.free"))
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "df_complex.free"))
assert ensure_always(
lambda: not has_datapoint_with_metric_name(backend, "memory.cached"
), 10)
assert ensure_always(
lambda: not has_datapoint_with_metric_name(backend, "uptime"), 5)
def test_docker_observer_labels():
"""
Test that docker observer picks up a fully configured endpoint from
container labels
"""
with run_agent(
dedent("""
observers:
- type: docker
""")) as [backend, _, _]:
with run_service(
"nginx",
name="nginx-disco-full",
labels={
"agent.signalfx.com.monitorType.80": "collectd/nginx",
"agent.signalfx.com.config.80.intervalSeconds": "1",
},
):
assert wait_for(
p(has_datapoint_with_dim, backend, "plugin",
"nginx")), "Didn't get nginx datapoints"
# Let nginx be removed by docker observer and collectd restart
time.sleep(5)
backend.datapoints.clear()
assert ensure_always(
lambda: not has_datapoint_with_dim(backend, "container_name",
"nginx-disco-full"), 10)
def test_docker_observer_labels_partial():
"""
Test that docker observer picks up a partially configured endpoint from
container labels
"""
with run_agent(
dedent("""
observers:
- type: docker
monitors:
- type: collectd/nginx
discoveryRule: container_name =~ "nginx-disco-partial" && port == 80
""")) as [backend, _, _]:
with run_service(
"nginx",
name="nginx-disco-partial",
labels={
"agent.signalfx.com.config.80.extraDimensions":
"{mydim: myvalue}"
},
):
assert wait_for(
p(has_datapoint_with_dim, backend, "plugin",
"nginx")), "Didn't get nginx datapoints"
assert wait_for(
p(has_datapoint_with_dim, backend, "mydim",
"myvalue")), "Didn't get extra dimension"
# Let nginx be removed by docker observer and collectd restart
time.sleep(5)
backend.datapoints.clear()
assert ensure_always(
lambda: not has_datapoint_with_dim(backend, "container_name",
"nginx-disco-partial"), 10)
def test_does_not_set_hostname_if_not_host_specific():
with run_agent("""
hostname: acmeinc.com
disableHostDimensions: true
monitors:
- type: collectd/signalfx-metadata
persistencePath: /dev/null
- type: collectd/cpu
- type: collectd/uptime
""") as [backend, _, _]:
assert ensure_always(
lambda: not has_datapoint_with_dim(backend, "host", "acmeinc.com")
), "Got overridden hostname in datapoint"
assert ensure_always(
lambda: not has_event_with_dim(backend, "host", "acmeinc.com")
), "Got overridden hostname in event"
def test_monitor_filter():
"""
Ensure the filters on monitors get applied
"""
with run_agent("""
monitors:
- type: collectd/signalfx-metadata
- type: collectd/df
- type: collectd/memory
metricsToExclude:
- metricName: memory.used
- type: collectd/uptime
""") as [backend, _, update_config]:
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "df_complex.free"))
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "memory.free"))
assert ensure_always(
lambda: not has_datapoint_with_metric_name(backend, "memory.used"))
update_config("""
monitors:
- type: collectd/signalfx-metadata
- type: collectd/df
- type: collectd/memory
- type: collectd/uptime
""")
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "memory.used"))
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "memory.free"))
def test_include_filter_with_monitor_type():
"""
Test that include filters will override exclude filters
"""
with run_agent("""
monitors:
- type: collectd/disk
- type: collectd/uptime
metricsToExclude:
- metricNames:
- disk_time.read
monitorType: collectd/disk
- metricNames:
- disk_ops.read
- disk_ops.write
monitorType: collectd/disk
negated: true
metricsToInclude:
- metricNames:
- disk_time.read
""") as [backend, _, _]:
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "disk_ops.read"))
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "disk_ops.write"),
5)
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "disk_time.read"),
5)
assert ensure_always(
lambda: not has_datapoint_with_metric_name(backend,
"disk_time.write"), 5)
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "uptime"), 5)
def test_basic_filtering():
with run_agent(BASIC_CONFIG) as [backend, _, _]:
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "uptime"))
assert ensure_always(
lambda: not has_datapoint_with_metric_name(backend,
"cpu.utilization"), 10)
def test_kubernetes_cluster_namespace_scope(agent_image, minikube,
k8s_namespace):
monitors = [{
"type": "kubernetes-cluster",
"kubernetesAPI": {
"authType": "serviceAccount"
},
"namespace": "good"
}]
with run_k8s_with_agent(
agent_image,
minikube,
monitors,
namespace=k8s_namespace,
yamls=[local_file("good-pod.yaml"),
local_file("bad-pod.yaml")],
) as [backend, _]:
assert wait_for(
p(has_datapoint,
backend,
dimensions={"kubernetes_namespace":
"good"})), "timed out waiting for good pod metrics"
assert ensure_always(lambda: not has_datapoint(
backend, dimensions={"kubernetes_namespace": "bad"})
), "got pod metrics from unspecified namespace"
def test_mixed_regex_and_non_regex_filters():
with run_agent("""
monitors:
- type: collectd/signalfx-metadata
- type: collectd/memory
- type: collectd/df
- type: collectd/uptime
metricsToExclude:
- metricNames:
- /memory.used/
- asdflkjassdf
negated: true
""") as [backend, _, _]:
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "memory.used"))
assert ensure_always(
lambda: not has_datapoint_with_metric_name(backend, "memory.free"),
10)
assert ensure_always(
lambda: not has_datapoint_with_metric_name(backend, "uptime"), 5)
def test_basic_service_discovery():
with run_agent(CONFIG) as [backend, get_output, _]:
with run_service("nginx", name="nginx-discovery"):
assert wait_for(
p(has_datapoint_with_dim, backend, "plugin",
"nginx")), "Didn't get nginx datapoints"
# Let nginx be removed by docker observer and collectd restart
time.sleep(5)
backend.datapoints.clear()
assert ensure_always(
lambda: not has_datapoint_with_dim(backend, "plugin", "nginx"), 10)
assert not has_log_message(get_output(), "error")
def test_docker_image_filtering():
with run_service("nginx") as nginx_container:
with run_agent(
"""
monitors:
- type: docker-container-stats
excludedImages:
- "%s"
"""
% nginx_container.attrs["Image"]
) as [backend, _, _]:
assert ensure_always(lambda: not has_datapoint_with_dim(backend, "container_id", nginx_container.id))
def test_docker_stops_watching_paused_containers():
with run_service("nginx") as nginx_container:
with run_agent(
"""
monitors:
- type: docker-container-stats
"""
) as [backend, _, _]:
assert wait_for(
p(has_datapoint_with_dim, backend, "container_id", nginx_container.id)
), "Didn't get nginx datapoints"
nginx_container.pause()
time.sleep(5)
backend.datapoints.clear()
assert ensure_always(lambda: not has_datapoint_with_dim(backend, "container_id", nginx_container.id))
def test_docker_observer():
with run_agent(CONFIG) as [backend, _, _]:
with run_service("nginx",
name="nginx-discovery",
labels={"mylabel": "abc"}):
assert wait_for(
p(has_datapoint_with_dim, backend, "plugin",
"nginx")), "Didn't get nginx datapoints"
assert wait_for(p(has_datapoint_with_dim, backend, "mydim",
"abc")), "Didn't get custom label dimension"
# Let nginx be removed by docker observer and collectd restart
time.sleep(5)
backend.datapoints.clear()
assert ensure_always(
lambda: not has_datapoint_with_dim(backend, "container_name",
"nginx-discovery"), 10)
def test_does_not_set_hostname_on_monitor_if_not_host_specific():
with run_agent("""
hostname: acmeinc.com
monitors:
- type: collectd/signalfx-metadata
persistencePath: /dev/null
- type: collectd/cpu
- type: collectd/uptime
disableHostDimensions: true
""") as [backend, _, _]:
assert wait_for(
p(has_datapoint_with_all_dims, backend,
dict(host="acmeinc.com", plugin="signalfx-metadata"))
), "Didn't get overridden hostname in datapoint"
assert ensure_always(lambda: not has_datapoint_with_dim(
backend, "uptime", "acmeinc.com")
), "Got overridden hostname in datapoint"
def test_custom_collectd_shutdown():
with run_agent(
dedent("""
monitors:
- type: collectd/df
- type: collectd/custom
template: |
LoadPlugin "ping"
<Plugin ping>
Host "google.com"
</Plugin>
""")) as [backend, _, configure]:
assert wait_for(p(has_datapoint_with_dim, backend, "plugin",
"ping")), "Didn't get ping datapoints"
assert wait_for(p(has_datapoint_with_dim, backend, "plugin",
"df")), "Didn't get df datapoints"
configure(
dedent("""
monitors:
- type: collectd/df
"""))
time.sleep(3)
backend.datapoints.clear()
assert ensure_always(
lambda: not has_datapoint_with_dim(backend, "plugin", "ping")
), "Got ping datapoint when we shouldn't have"
configure(
dedent("""
monitors:
- type: collectd/df
- type: collectd/custom
template: |
LoadPlugin "ping"
<Plugin ping>
Host "google.com"
</Plugin>
"""))
assert wait_for(p(has_datapoint_with_dim, backend, "plugin",
"ping")), "Didn't get ping datapoints"
def test_solr_monitor():
with run_service("solr") as solr_container:
host = container_ip(solr_container)
config = dedent(f"""
monitors:
- type: collectd/solr
host: {host}
port: 8983
""")
assert wait_for(p(tcp_socket_open, host, 8983),
60), "service not listening on port"
with run_agent(config) as [backend, get_output, _]:
assert wait_for(
p(has_datapoint_with_dim, backend, "plugin",
"solr")), "Didn't get solr datapoints"
assert ensure_always(lambda: has_datapoint_with_metric_name(
backend, "counter.solr.http_5xx_responses"))
assert not has_log_message(get_output().lower(),
"error"), "error found in agent output!"
def test_k8s_events_without_whitelist(agent_image, minikube, k8s_observer,
k8s_namespace):
monitors = [{
"type": "kubernetes-events",
"kubernetesAPI": {
"authType": "serviceAccount"
}
}]
with run_k8s_with_agent(agent_image,
minikube,
monitors,
observer=k8s_observer,
namespace=k8s_namespace) as [
backend,
agent,
]:
assert ensure_always(lambda: not backend.events, 30), (
"event received!\n\nAGENT STATUS:\n%s\n\nAGENT CONTAINER LOGS:\n%s\n"
% (agent.get_status(), agent.get_container_logs()))
def test_overlapping_filter_with_monitor_type2():
"""
Test overlapping filters with different negation. Blacklist is favored
"""
with run_agent("""
monitors:
- type: collectd/memory
- type: collectd/uptime
metricsToExclude:
- metricName: uptime
monitorType: collectd/uptime
- metricName: uptime
negated: true
monitorType: collectd/uptime
""") as [backend, _, _]:
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "memory.used"))
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "memory.free"))
assert ensure_always(
lambda: not has_datapoint_with_metric_name(backend, "uptime"), 5)
def test_docker_observer_labels_multiple_monitors_per_port():
"""
Test that we can configure multiple monitors per port using labels
"""
with run_agent(
dedent("""
observers:
- type: docker
""")) as [backend, _, _]:
with run_service(
"nginx",
name="nginx-multi-monitors",
labels={
"agent.signalfx.com.monitorType.80":
"collectd/nginx",
"agent.signalfx.com.config.80.intervalSeconds":
"1",
"agent.signalfx.com.config.80.extraDimensions":
"{app: nginx}",
"agent.signalfx.com.monitorType.80-nginx2":
"collectd/nginx",
"agent.signalfx.com.config.80-nginx2.intervalSeconds":
"1",
"agent.signalfx.com.config.80-nginx2.extraDimensions":
"{app: other}",
},
):
assert wait_for(
p(has_datapoint_with_dim, backend, "plugin",
"nginx")), "Didn't get nginx datapoints"
assert wait_for(p(has_datapoint_with_dim, backend, "app",
"nginx")), "Didn't get extra dims"
assert wait_for(p(has_datapoint_with_dim, backend, "app",
"other")), "Didn't get extra dims"
# Let nginx be removed by docker observer and collectd restart
time.sleep(5)
backend.datapoints.clear()
assert ensure_always(
lambda: not has_datapoint_with_dim(backend, "container_name",
"nginx-multi-monitors"), 10)
def test_negated_filtering():
with run_agent(NEGATIVE_FILTERING_CONFIG) as [backend, _, _]:
assert wait_for(
lambda: has_datapoint_with_metric_name(backend, "memory.used"))
assert ensure_always(
lambda: not has_datapoint_with_metric_name(backend, "uptime"), 10)
