def test_mem_free_from_memfree_when_memavailable_not_mentioned(
elasticapm_client, tmpdir):
proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
proc_stat = os.path.join(tmpdir.strpath, "stat")
proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
for path, content in (
(proc_stat, TEMPLATE_PROC_STAT_DEBIAN.format(user=0, idle=0)),
(proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
(proc_meminfo, TEMPLATE_PROC_MEMINFO_NO_MEMAVAILABLE),
):
with open(path, mode="w") as f:
f.write(content)
metricset = CPUMetricSet(
MetricsRegistry(elasticapm_client),
sys_stats_file=proc_stat,
process_stats_file=proc_stat_self,
memory_stats_file=proc_meminfo,
)
for path, content in (
(proc_stat, TEMPLATE_PROC_STAT_DEBIAN.format(user=400000,
idle=600000)),
(proc_stat_self,
TEMPLATE_PROC_STAT_SELF.format(utime=100000, stime=100000)),
(proc_meminfo, TEMPLATE_PROC_MEMINFO_NO_MEMAVAILABLE),
):
with open(path, mode="w") as f:
f.write(content)
data = next(metricset.collect())
mem_free_expected = sum([359184, 891296, 6416340
]) * 1024 # MemFree + Buffers + Cached, in bytes
assert data["samples"]["system.memory.actual.free"][
"value"] == mem_free_expected
def test_cpu_usage_when_cpu_total_is_zero(elasticapm_client, tmpdir):
proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
proc_stat = os.path.join(tmpdir.strpath, "stat")
proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
for path, content in (
(proc_stat, TEMPLATE_PROC_STAT_DEBIAN.format(user=0, idle=0)),
(proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
(proc_meminfo, TEMPLATE_PROC_MEMINFO_NO_MEMAVAILABLE),
):
with open(path, mode="w") as f:
f.write(content)
metricset = CPUMetricSet(
MetricsRegistry(elasticapm_client),
sys_stats_file=proc_stat,
process_stats_file=proc_stat_self,
memory_stats_file=proc_meminfo,
)
data = next(metricset.collect())
cpu_total_expected = 0
assert data["samples"]["system.cpu.total.norm.pct"][
"value"] == cpu_total_expected
assert data["samples"]["system.process.cpu.total.norm.pct"][
"value"] == cpu_total_expected
def test_metrics_not_collected_if_zero_and_reset():
m = MetricsSet(MetricsRegistry(0, lambda x: None))
counter = m.counter("counter", reset_on_collect=False)
resetting_counter = m.counter("resetting_counter", reset_on_collect=True)
gauge = m.gauge("gauge", reset_on_collect=False)
resetting_gauge = m.gauge("resetting_gauge", reset_on_collect=True)
timer = m.timer("timer", reset_on_collect=False)
resetting_timer = m.timer("resetting_timer", reset_on_collect=True)
counter.inc(), resetting_counter.inc()
gauge.val = 5
resetting_gauge.val = 5
timer.update(1, 1)
resetting_timer.update(1, 1)
data = list(m.collect())
more_data = list(m.collect())
assert set(data[0]["samples"].keys()) == {
"counter",
"resetting_counter",
"gauge",
"resetting_gauge",
"timer.count",
"timer.sum.us",
"resetting_timer.count",
"resetting_timer.sum.us",
}
assert set(more_data[0]["samples"].keys()) == {"counter", "gauge", "timer.count", "timer.sum.us"}
def test_cpu_mem_from_proc(elasticapm_client, proc_stat_template, tmpdir):
proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
proc_stat = os.path.join(tmpdir.strpath, "stat")
proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
for path, content in (
(proc_stat, proc_stat_template.format(user=0, idle=0)),
(proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
(proc_meminfo, TEMPLATE_PROC_MEMINFO),
):
with open(path, mode="w") as f:
f.write(content)
metricset = CPUMetricSet(
MetricsRegistry(elasticapm_client),
sys_stats_file=proc_stat,
process_stats_file=proc_stat_self,
memory_stats_file=proc_meminfo,
)
for path, content in (
(proc_stat, proc_stat_template.format(user=400000, idle=600000)),
(proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=100000, stime=100000)),
(proc_meminfo, TEMPLATE_PROC_MEMINFO),
):
with open(path, mode="w") as f:
f.write(content)
data = next(metricset.collect())
assert data["samples"]["system.cpu.total.norm.pct"]["value"] == 0.4
assert data["samples"]["system.process.cpu.total.norm.pct"]["value"] == 0.2
assert data["samples"]["system.memory.total"]["value"] == 16552841216
assert data["samples"]["system.memory.actual.free"]["value"] == 6670774272
assert data["samples"]["system.process.memory.rss.bytes"]["value"] == 47738880
assert data["samples"]["system.process.memory.size"]["value"] == 3686981632
def test_summary(elasticapm_client, prometheus):
metricset = PrometheusMetrics(MetricsRegistry(elasticapm_client))
summary = prometheus_client.Summary("a_bare_summary", "Bare summary")
summary_with_labels = prometheus_client.Summary("summary_with_labels",
"Summary with labels",
["alabel", "anotherlabel"])
summary.observe(5)
summary.observe(7)
summary.observe(9)
summary_with_labels.labels(alabel="foo", anotherlabel="baz").observe(7)
summary_with_labels.labels(alabel="bar",
anotherlabel="bazzinga").observe(11)
summary_with_labels.labels(alabel="foo", anotherlabel="baz").observe(2)
data = list(metricset.collect())
assert len(data) == 3
assert data[0]["samples"]["prometheus.metrics.a_bare_summary.count"][
"value"] == 3.0
assert data[0]["samples"]["prometheus.metrics.a_bare_summary.sum"][
"value"] == 21
assert data[1]["samples"]["prometheus.metrics.summary_with_labels.count"][
"value"] == 2.0
assert data[1]["samples"]["prometheus.metrics.summary_with_labels.sum"][
"value"] == 9.0
assert data[1]["tags"] == {"alabel": "foo", "anotherlabel": "baz"}
assert data[2]["samples"]["prometheus.metrics.summary_with_labels.count"][
"value"] == 1.0
assert data[2]["samples"]["prometheus.metrics.summary_with_labels.sum"][
"value"] == 11.0
assert data[2]["tags"] == {"alabel": "bar", "anotherlabel": "bazzinga"}
def test_metrics_registry():
mock_queue = mock.Mock()
registry = MetricsRegistry(0.001, queue_func=mock_queue)
registry.register("tests.metrics.base_tests.DummyMetricSet")
try:
registry.start_thread()
time.sleep(0.1)
assert mock_queue.call_count > 0
finally:
registry.stop_thread()
def test_compare_psutil_linux_metricsets(elasticapm_client):
psutil_metricset = cpu_psutil.CPUMetricSet(
MetricsRegistry(elasticapm_client))
linux_metricset = cpu_linux.CPUMetricSet(
MetricsRegistry(elasticapm_client))
# do something that generates some CPU load
for i in compat.irange(10**6):
j = i * i
psutil_data = next(psutil_metricset.collect())
linux_data = next(linux_metricset.collect())
assert (abs(psutil_data["samples"]["system.cpu.total.norm.pct"]["value"] -
linux_data["samples"]["system.cpu.total.norm.pct"]["value"]) <
0.02)
assert (abs(
psutil_data["samples"]["system.process.cpu.total.norm.pct"]["value"] -
linux_data["samples"]["system.process.cpu.total.norm.pct"]["value"]) <
0.02)
def test_cpu_mem_from_proc(elasticapm_client, proc_stat_template, tmpdir):
proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
proc_stat = os.path.join(tmpdir.strpath, "stat")
proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
cgroup_memory_limit = os.path.join(tmpdir.strpath, "memory",
"memory.limit_in_bytes")
cgroup_memory_usage = os.path.join(tmpdir.strpath, "memory",
"memory.usage_in_bytes")
proc_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
os.mkdir(os.path.join(tmpdir.strpath, "memory"))
proc_self_mount = os.path.join(tmpdir.strpath, "mountinfo")
for path, content in (
(proc_stat, proc_stat_template.format(user=0, idle=0)),
(proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
(proc_meminfo, TEMPLATE_PROC_MEMINFO),
(cgroup_memory_limit, TEMPLATE_CGROUP_MEM_LIMIT_IN_BYTES),
(cgroup_memory_usage, TEMPLATE_CGROUP_MEM_USAGE_IN_BYTES),
(proc_self_cgroup, "9:memory:/slice"),
(
proc_self_mount,
"39 30 0:35 / " + tmpdir.strpath +
"/memory rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n",
),
):
with open(path, mode="w") as f:
f.write(content)
metricset = CPUMetricSet(
MetricsRegistry(elasticapm_client),
sys_stats_file=proc_stat,
process_stats_file=proc_stat_self,
memory_stats_file=proc_meminfo,
proc_self_cgroup=proc_self_cgroup,
mount_info=proc_self_mount,
)
for path, content in (
(proc_stat, proc_stat_template.format(user=400000, idle=600000)),
(proc_stat_self,
TEMPLATE_PROC_STAT_SELF.format(utime=100000, stime=100000)),
(proc_meminfo, TEMPLATE_PROC_MEMINFO),
):
with open(path, mode="w") as f:
f.write(content)
data = next(metricset.collect())
assert data["samples"]["system.cpu.total.norm.pct"]["value"] == 0.4
assert data["samples"]["system.process.cpu.total.norm.pct"]["value"] == 0.2
assert data["samples"]["system.memory.total"]["value"] == 16552841216
assert data["samples"]["system.memory.actual.free"]["value"] == 6670774272
assert data["samples"]["system.process.memory.rss.bytes"][
"value"] == 47738880
assert data["samples"]["system.process.memory.size"]["value"] == 3686981632
def test_mem_from_cgroup2(elasticapm_client, tmpdir):
proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
proc_stat = os.path.join(tmpdir.strpath, "stat")
proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
cgroup2_memory_limit = os.path.join(tmpdir.strpath, "slice", "memory.max")
cgroup2_memory_usage = os.path.join(tmpdir.strpath, "slice",
"memory.current")
cgroup2_memory_stat = os.path.join(tmpdir.strpath, "slice", "memory.stat")
cgroup2_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
proc_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
os.mkdir(os.path.join(tmpdir.strpath, "slice"))
proc_self_mount = os.path.join(tmpdir.strpath, "mountinfo")
for path, content in (
(proc_stat, TEMPLATE_PROC_STAT_DEBIAN.format(user=0, idle=0)),
(proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
(proc_meminfo, TEMPLATE_PROC_MEMINFO),
(cgroup2_memory_limit, TEMPLATE_CGROUP_MEM_LIMIT_IN_BYTES_LIMITED),
(cgroup2_memory_usage, TEMPLATE_CGROUP_MEM_USAGE_IN_BYTES),
(cgroup2_memory_stat, TEMPLATE_CGROUP_MEM_STAT),
(cgroup2_self_cgroup, "9:memory:/slice"),
(
proc_self_mount,
"30 23 0:26 / " + tmpdir.strpath +
" rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup rw,seclabel\n",
),
):
with open(path, mode="w") as f:
f.write(content)
metricset = CPUMetricSet(
MetricsRegistry(elasticapm_client),
sys_stats_file=proc_stat,
process_stats_file=proc_stat_self,
memory_stats_file=proc_meminfo,
proc_self_cgroup=proc_self_cgroup,
mount_info=proc_self_mount,
)
data = next(metricset.collect())
assert data["samples"]["system.memory.total"]["value"] == 16552841216
assert data["samples"]["system.memory.actual.free"]["value"] == 6670774272
assert data["samples"]["system.process.memory.rss.bytes"][
"value"] == 47738880
assert data["samples"]["system.process.memory.size"]["value"] == 3686981632
assert data["samples"]["system.process.cgroup.memory.mem.limit.bytes"][
"value"] == 7964778496
assert data["samples"]["system.process.cgroup.memory.mem.usage.bytes"][
"value"] == 954370560
assert data["samples"][
"system.process.cgroup.memory.stats.inactive_file.bytes"][
"value"] == 10407936
def test_metrics_registry():
mock_queue = mock.Mock()
registry = MetricsRegistry(0.001, queue_func=mock_queue)
registry.register("tests.metrics.base_tests.DummyMetricSet")
time.sleep(0.1)
assert mock_queue.call_count > 0
registry._stop_collect_timer()
def test_mem_from_cgroup_files_dont_exist(elasticapm_client, tmpdir):
# it appears that on Google App engine, there is a possibility of
# memory.limit_in_bytes existing while memory.usage_in_bytes does not.
# See https://github.com/elastic/apm-agent-python/issues/985
proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
proc_stat = os.path.join(tmpdir.strpath, "stat")
proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
cgroup_memory_limit = os.path.join(tmpdir.strpath, "memory",
"memory.limit_in_bytes")
# intentionally commented out
# cgroup_memory_usage = os.path.join(tmpdir.strpath, "memory", "memory.usage_in_bytes")
cgroup_memory_stat = os.path.join(tmpdir.strpath, "memory", "memory.stat")
proc_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
os.mkdir(os.path.join(tmpdir.strpath, "memory"))
proc_self_mount = os.path.join(tmpdir.strpath, "mountinfo")
for path, content in (
(proc_stat, TEMPLATE_PROC_STAT_DEBIAN.format(user=0, idle=0)),
(proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
(proc_meminfo, TEMPLATE_PROC_MEMINFO),
(cgroup_memory_limit, TEMPLATE_CGROUP_MEM_LIMIT_IN_BYTES_LIMITED),
# intentionally commented out
# (cgroup_memory_usage, TEMPLATE_CGROUP_MEM_USAGE_IN_BYTES),
(cgroup_memory_stat, TEMPLATE_CGROUP_MEM_STAT),
(proc_self_cgroup, "9:memory:/slice"),
(
proc_self_mount,
"39 30 0:35 / " + tmpdir.strpath +
"/memory rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n",
),
):
with open(path, mode="w") as f:
f.write(content)
metricset = CPUMetricSet(
MetricsRegistry(elasticapm_client),
sys_stats_file=proc_stat,
process_stats_file=proc_stat_self,
memory_stats_file=proc_meminfo,
proc_self_cgroup=proc_self_cgroup,
mount_info=proc_self_mount,
)
assert metricset.cgroup_files.limit is not None
assert metricset.cgroup_files.usage is None
data = next(metricset.collect())
assert "system.process.cgroup.memory.mem.limit.bytes" in data["samples"]
assert "system.process.cgroup.memory.mem.usage.bytes" not in data[
"samples"]
def test_metrics_counter(elasticapm_client):
metricset = MetricsSet(MetricsRegistry(elasticapm_client))
metricset.counter("x").inc()
data = next(metricset.collect())
assert data["samples"]["x"]["value"] == 1
metricset.counter("x").inc(10)
data = next(metricset.collect())
assert data["samples"]["x"]["value"] == 11
metricset.counter("x").dec(10)
data = next(metricset.collect())
assert data["samples"]["x"]["value"] == 1
metricset.counter("x").dec()
data = next(metricset.collect())
assert data["samples"]["x"]["value"] == 0
def test_metrics_multithreaded():
metricset = MetricsSet(MetricsRegistry(0, lambda x: None))
pool = Pool(5)
def target():
for i in range(500):
metricset.counter("x").inc(i + 1)
time.sleep(0.0000001)
[pool.apply_async(target, ()) for i in range(10)]
pool.close()
pool.join()
expected = 10 * ((500 * 501) / 2)
assert metricset.counter("x").val == expected
def test_metrics_counter():
metricset = MetricsSet(MetricsRegistry(0, lambda x: None))
metricset.counter("x").inc()
data = next(metricset.collect())
assert data["samples"]["x"]["value"] == 1
metricset.counter("x").inc(10)
data = next(metricset.collect())
assert data["samples"]["x"]["value"] == 11
metricset.counter("x").dec(10)
data = next(metricset.collect())
assert data["samples"]["x"]["value"] == 1
metricset.counter("x").dec()
data = next(metricset.collect())
assert data["samples"]["x"]["value"] == 0
def test_histogram(elasticapm_client, prometheus):
metricset = PrometheusMetrics(MetricsRegistry(elasticapm_client))
histo = prometheus_client.Histogram("histo",
"test histogram",
buckets=[1, 10, 100,
float("inf")])
histo_with_labels = prometheus_client.Histogram(
"histowithlabel",
"test histogram with labels", ["alabel", "anotherlabel"],
buckets=[1, 10, 100, float("inf")])
histo.observe(0.5)
histo.observe(0.6)
histo.observe(1.5)
histo.observe(26)
histo.observe(42)
histo.observe(12)
histo.observe(105)
histo_with_labels.labels(alabel="foo", anotherlabel="baz").observe(1)
histo_with_labels.labels(alabel="foo", anotherlabel="baz").observe(10)
histo_with_labels.labels(alabel="foo", anotherlabel="baz").observe(100)
histo_with_labels.labels(alabel="foo",
anotherlabel="bazzinga").observe(1000)
data = list(metricset.collect())
assert data[0]["samples"]["prometheus.metrics.histo"]["values"] == [
0.5, 5.5, 55.0, 100.0
]
assert data[0]["samples"]["prometheus.metrics.histo"]["counts"] == [
2, 1, 3, 1
]
assert all(
isinstance(v, int)
for v in data[0]["samples"]["prometheus.metrics.histo"]["counts"])
assert data[1]["samples"]["prometheus.metrics.histowithlabel"][
"values"] == [0.5, 5.5, 55.0, 100.0]
assert data[1]["samples"]["prometheus.metrics.histowithlabel"][
"counts"] == [1, 1, 1, 0]
assert all(
isinstance(v, int) for v in data[1]["samples"]
["prometheus.metrics.histowithlabel"]["counts"])
assert data[1]["tags"] == {"alabel": "foo", "anotherlabel": "baz"}
assert data[2]["samples"]["prometheus.metrics.histowithlabel"][
"values"] == [0.5, 5.5, 55.0, 100.0]
assert data[2]["samples"]["prometheus.metrics.histowithlabel"][
"counts"] == [0, 0, 0, 1]
assert all(
isinstance(v, int) for v in data[2]["samples"]
["prometheus.metrics.histowithlabel"]["counts"])
assert data[2]["tags"] == {"alabel": "foo", "anotherlabel": "bazzinga"}
def test_metric_limit(caplog, elasticapm_client):
m = MetricsSet(MetricsRegistry(elasticapm_client))
with caplog.at_level(logging.WARNING, logger="elasticapm.metrics"):
for i in range(2):
counter = m.counter("counter", some_label=i)
gauge = m.gauge("gauge", some_label=i)
timer = m.timer("timer", some_label=i)
if i == 0:
assert isinstance(timer, Timer)
assert isinstance(gauge, Gauge)
assert isinstance(counter, Counter)
else:
assert isinstance(timer, NoopMetric)
assert isinstance(gauge, NoopMetric)
assert isinstance(counter, NoopMetric)
assert_any_record_contains(caplog.records, "The limit of 3 metricsets has been reached", "elasticapm.metrics")
def test_metrics_histogram(elasticapm_client):
metricset = MetricsSet(MetricsRegistry(elasticapm_client))
hist = metricset.histogram("x", buckets=[1, 10, 100])
assert len(hist.buckets) == 4
hist.update(0.3)
hist.update(1)
hist.update(5)
hist.update(20)
hist.update(100)
hist.update(1000)
data = list(metricset.collect())
assert len(data) == 1
d = data[0]
assert d["samples"]["x"]["counts"] == [2, 1, 2, 1]
assert d["samples"]["x"]["values"] == [0.5, 5.5, 55.0, 100]
def test_mem_from_cgroup2_max_handling(elasticapm_client, tmpdir):
proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
proc_stat = os.path.join(tmpdir.strpath, "stat")
proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
cgroup2_memory_limit = os.path.join(tmpdir.strpath, "slice", "memory.max")
cgroup2_memory_usage = os.path.join(tmpdir.strpath, "slice",
"memory.current")
cgroup2_memory_stat = os.path.join(tmpdir.strpath, "slice", "memory.stat")
cgroup2_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
proc_self_cgroup = os.path.join(tmpdir.strpath, "cgroup")
os.mkdir(os.path.join(tmpdir.strpath, "slice"))
proc_self_mount = os.path.join(tmpdir.strpath, "mountinfo")
for path, content in (
(proc_stat, TEMPLATE_PROC_STAT_DEBIAN.format(user=0, idle=0)),
(proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
(proc_meminfo, TEMPLATE_PROC_MEMINFO),
(cgroup2_memory_limit, "max"),
(cgroup2_memory_usage, TEMPLATE_CGROUP_MEM_USAGE_IN_BYTES),
(cgroup2_memory_stat, TEMPLATE_CGROUP_MEM_STAT),
(cgroup2_self_cgroup, "9:memory:/slice"),
(
proc_self_mount,
"30 23 0:26 / " + tmpdir.strpath +
" rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup rw,seclabel\n",
),
):
with open(path, mode="w") as f:
f.write(content)
metricset = CPUMetricSet(
MetricsRegistry(elasticapm_client),
sys_stats_file=proc_stat,
process_stats_file=proc_stat_self,
memory_stats_file=proc_meminfo,
proc_self_cgroup=proc_self_cgroup,
mount_info=proc_self_mount,
)
data = next(metricset.collect())
assert "system.process.cgroup.memory.mem.limit.bytes" not in data[
"samples"]
assert "system.process.cgroup.memory.mem.usage.bytes" not in data[
"samples"]
assert "system.process.cgroup.memory.stats.inactive_file.bytes" not in data[
"samples"]
def test_cpu_mem_from_psutil(elasticapm_client):
metricset = cpu_psutil.CPUMetricSet(MetricsRegistry(elasticapm_client))
# do something that generates some CPU load
for i in compat.irange(10**6):
j = i * i
data = next(metricset.collect())
# we can't really test any specific values here as it depends on the system state.
# Mocking is also not really a viable choice, as we would then lose the "integration testing"
# nature of this test with different versions of psutil
assert 0 < data["samples"]["system.cpu.total.norm.pct"]["value"] < 1
assert 0 < data["samples"]["system.process.cpu.total.norm.pct"]["value"] < 1
assert data["samples"]["system.memory.total"]["value"] > 0
assert data["samples"]["system.memory.actual.free"]["value"] > 0
assert data["samples"]["system.process.memory.rss.bytes"]["value"] > 0
assert data["samples"]["system.process.memory.size"]["value"] > 0
def test_metric_limit(caplog):
m = MetricsSet(MetricsRegistry(0, lambda x: None))
with caplog.at_level(logging.WARNING, logger="elasticapm.metrics"):
for i in range(2):
counter = m.counter("counter", some_label=i)
gauge = m.gauge("gauge", some_label=i)
timer = m.timer("timer", some_label=i)
if i == 0:
assert isinstance(timer, Timer)
assert isinstance(gauge, Gauge)
assert isinstance(counter, Counter)
else:
assert isinstance(timer, NoopMetric)
assert isinstance(gauge, NoopMetric)
assert isinstance(counter, NoopMetric)
assert len(caplog.records) == 1
record = caplog.records[0]
assert "The limit of 3 metricsets has been reached" in record.message
def test_counter(elasticapm_client, prometheus):
metricset = PrometheusMetrics(MetricsRegistry(elasticapm_client))
counter = prometheus_client.Counter("a_bare_counter", "Bare counter")
counter_with_labels = prometheus_client.Counter("counter_with_labels",
"Counter with labels",
["alabel", "anotherlabel"])
counter.inc()
counter_with_labels.labels(alabel="foo", anotherlabel="baz").inc()
counter_with_labels.labels(alabel="bar", anotherlabel="bazzinga").inc()
counter_with_labels.labels(alabel="foo", anotherlabel="baz").inc()
data = list(metricset.collect())
assert len(data) == 3
assert data[0]["samples"]["prometheus.metrics.a_bare_counter"][
"value"] == 1.0
assert data[1]["samples"]["prometheus.metrics.counter_with_labels"][
"value"] == 2.0
assert data[1]["tags"] == {"alabel": "foo", "anotherlabel": "baz"}
assert data[2]["samples"]["prometheus.metrics.counter_with_labels"][
"value"] == 1.0
assert data[2]["tags"] == {"alabel": "bar", "anotherlabel": "bazzinga"}
def test_gauge(elasticapm_client, prometheus):
metricset = PrometheusMetrics(MetricsRegistry(elasticapm_client))
gauge = prometheus_client.Gauge("a_bare_gauge", "Bare gauge")
gauge_with_labels = prometheus_client.Gauge("gauge_with_labels",
"Gauge with labels",
["alabel", "anotherlabel"])
gauge.set(5)
gauge_with_labels.labels(alabel="foo", anotherlabel="baz").set(7)
gauge_with_labels.labels(alabel="bar", anotherlabel="bazzinga").set(11)
gauge_with_labels.labels(alabel="foo", anotherlabel="baz").set(2)
data = list(metricset.collect())
assert len(data) == 3
assert data[0]["samples"]["prometheus.metrics.a_bare_gauge"][
"value"] == 5.0
assert data[1]["samples"]["prometheus.metrics.gauge_with_labels"][
"value"] == 2.0
assert data[1]["tags"] == {"alabel": "foo", "anotherlabel": "baz"}
assert data[2]["samples"]["prometheus.metrics.gauge_with_labels"][
"value"] == 11.0
assert data[2]["tags"] == {"alabel": "bar", "anotherlabel": "bazzinga"}
def test_metrics_labels():
metricset = MetricsSet(MetricsRegistry(0, lambda x: None))
metricset.counter("x", mylabel="a").inc()
metricset.counter("y", mylabel="a").inc()
metricset.counter("x", mylabel="b").inc().inc()
metricset.counter("x", mylabel="b", myotherlabel="c").inc()
metricset.counter("x", mylabel="a").dec()
data = list(metricset.collect())
asserts = 0
for d in data:
if d["tags"] == {"mylabel": "a"}:
assert d["samples"]["x"]["value"] == 0
assert d["samples"]["y"]["value"] == 1
asserts += 1
elif d["tags"] == {"mylabel": "b"}:
assert d["samples"]["x"]["value"] == 2
asserts += 1
elif d["tags"] == {"mylabel": "b", "myotherlabel": "c"}:
assert d["samples"]["x"]["value"] == 1
asserts += 1
assert asserts == 3
def __init__(self, config=None, **inline):
# configure loggers first
cls = self.__class__
self.logger = get_logger("%s.%s" % (cls.__module__, cls.__name__))
self.error_logger = get_logger("elasticapm.errors")
self.tracer = None
self.processors = []
self.filter_exception_types_dict = {}
self._service_info = None
config = Config(config, inline_dict=inline)
if config.errors:
for msg in config.errors.values():
self.error_logger.error(msg)
config.disable_send = True
self.config = VersionedConfig(config, version=None)
# Insert the log_record_factory into the logging library
# The LogRecordFactory functionality is only available on python 3.2+
if compat.PY3 and not self.config.disable_log_record_factory:
record_factory = logging.getLogRecordFactory()
# Only way to know if it's wrapped is to create a log record
throwaway_record = record_factory(__name__, logging.DEBUG, __file__, 252, "dummy_msg", [], None)
if not hasattr(throwaway_record, "elasticapm_labels"):
self.logger.debug("Inserting elasticapm log_record_factory into logging")
# Late import due to circular imports
import elasticapm.handlers.logging as elastic_logging
new_factory = elastic_logging.log_record_factory(record_factory)
logging.setLogRecordFactory(new_factory)
headers = {
"Content-Type": "application/x-ndjson",
"Content-Encoding": "gzip",
"User-Agent": "elasticapm-python/%s" % elasticapm.VERSION,
}
if self.config.secret_token:
headers["Authorization"] = "Bearer %s" % self.config.secret_token
transport_kwargs = {
"metadata": self._build_metadata(),
"headers": headers,
"verify_server_cert": self.config.verify_server_cert,
"server_cert": self.config.server_cert,
"timeout": self.config.server_timeout,
"max_flush_time": self.config.api_request_time / 1000.0,
"max_buffer_size": self.config.api_request_size,
"processors": self.load_processors(),
}
self._api_endpoint_url = compat.urlparse.urljoin(
self.config.server_url if self.config.server_url.endswith("/") else self.config.server_url + "/",
constants.EVENTS_API_PATH,
)
self._transport = import_string(self.config.transport_class)(self._api_endpoint_url, **transport_kwargs)
for exc_to_filter in self.config.filter_exception_types or []:
exc_to_filter_type = exc_to_filter.split(".")[-1]
exc_to_filter_module = ".".join(exc_to_filter.split(".")[:-1])
self.filter_exception_types_dict[exc_to_filter_type] = exc_to_filter_module
if platform.python_implementation() == "PyPy":
# PyPy introduces a `_functools.partial.__call__` frame due to our use
# of `partial` in AbstractInstrumentedModule
skip_modules = ("elasticapm.", "_functools")
else:
skip_modules = ("elasticapm.",)
self.tracer = Tracer(
frames_collector_func=lambda: list(
stacks.iter_stack_frames(
start_frame=inspect.currentframe(), skip_top_modules=skip_modules, config=self.config
)
),
frames_processing_func=lambda frames: self._get_stack_info_for_trace(
frames,
library_frame_context_lines=self.config.source_lines_span_library_frames,
in_app_frame_context_lines=self.config.source_lines_span_app_frames,
with_locals=self.config.collect_local_variables in ("all", "transactions"),
locals_processor_func=lambda local_var: varmap(
lambda k, v: shorten(
v,
list_length=self.config.local_var_list_max_length,
string_length=self.config.local_var_max_length,
dict_length=self.config.local_var_dict_max_length,
),
local_var,
),
),
queue_func=self.queue,
config=self.config,
agent=self,
)
self.include_paths_re = stacks.get_path_regex(self.config.include_paths) if self.config.include_paths else None
self.exclude_paths_re = stacks.get_path_regex(self.config.exclude_paths) if self.config.exclude_paths else None
self._metrics = MetricsRegistry(
self.config.metrics_interval / 1000.0, self.queue, ignore_patterns=self.config.disable_metrics
)
for path in self.config.metrics_sets:
self._metrics.register(path)
if self.config.breakdown_metrics:
self._metrics.register("elasticapm.metrics.sets.breakdown.BreakdownMetricSet")
compat.atexit_register(self.close)
if self.config.central_config:
self._config_updater = IntervalTimer(
update_config, 1, "eapm conf updater", daemon=True, args=(self,), evaluate_function_interval=True
)
self._config_updater.start()
else:
self._config_updater = None
class Client(object):
"""
The base ElasticAPM client, which handles communication over the
HTTP API to the APM Server.
Will read default configuration from the environment variable
``ELASTIC_APM_APP_NAME`` and ``ELASTIC_APM_SECRET_TOKEN``
if available. ::
>>> from elasticapm import Client
>>> # Read configuration from environment
>>> client = Client()
>>> # Configure the client manually
>>> client = Client(
>>> include_paths=['my.package'],
>>> service_name='myapp',
>>> secret_token='secret_token',
>>> )
>>> # Record an exception
>>> try:
>>> 1/0
>>> except ZeroDivisionError:
>>> ident = client.capture_exception()
>>> print ("Exception caught; reference is %%s" %% ident)
"""
logger = get_logger("elasticapm")
def __init__(self, config=None, **inline):
# configure loggers first
cls = self.__class__
self.logger = get_logger("%s.%s" % (cls.__module__, cls.__name__))
self.error_logger = get_logger("elasticapm.errors")
self.tracer = None
self.processors = []
self.filter_exception_types_dict = {}
self._service_info = None
config = Config(config, inline_dict=inline)
if config.errors:
for msg in config.errors.values():
self.error_logger.error(msg)
config.disable_send = True
self.config = VersionedConfig(config, version=None)
# Insert the log_record_factory into the logging library
# The LogRecordFactory functionality is only available on python 3.2+
if compat.PY3 and not self.config.disable_log_record_factory:
record_factory = logging.getLogRecordFactory()
# Only way to know if it's wrapped is to create a log record
throwaway_record = record_factory(__name__, logging.DEBUG, __file__, 252, "dummy_msg", [], None)
if not hasattr(throwaway_record, "elasticapm_labels"):
self.logger.debug("Inserting elasticapm log_record_factory into logging")
# Late import due to circular imports
import elasticapm.handlers.logging as elastic_logging
new_factory = elastic_logging.log_record_factory(record_factory)
logging.setLogRecordFactory(new_factory)
headers = {
"Content-Type": "application/x-ndjson",
"Content-Encoding": "gzip",
"User-Agent": "elasticapm-python/%s" % elasticapm.VERSION,
}
if self.config.secret_token:
headers["Authorization"] = "Bearer %s" % self.config.secret_token
transport_kwargs = {
"metadata": self._build_metadata(),
"headers": headers,
"verify_server_cert": self.config.verify_server_cert,
"server_cert": self.config.server_cert,
"timeout": self.config.server_timeout,
"max_flush_time": self.config.api_request_time / 1000.0,
"max_buffer_size": self.config.api_request_size,
"processors": self.load_processors(),
}
self._api_endpoint_url = compat.urlparse.urljoin(
self.config.server_url if self.config.server_url.endswith("/") else self.config.server_url + "/",
constants.EVENTS_API_PATH,
)
self._transport = import_string(self.config.transport_class)(self._api_endpoint_url, **transport_kwargs)
for exc_to_filter in self.config.filter_exception_types or []:
exc_to_filter_type = exc_to_filter.split(".")[-1]
exc_to_filter_module = ".".join(exc_to_filter.split(".")[:-1])
self.filter_exception_types_dict[exc_to_filter_type] = exc_to_filter_module
if platform.python_implementation() == "PyPy":
# PyPy introduces a `_functools.partial.__call__` frame due to our use
# of `partial` in AbstractInstrumentedModule
skip_modules = ("elasticapm.", "_functools")
else:
skip_modules = ("elasticapm.",)
self.tracer = Tracer(
frames_collector_func=lambda: list(
stacks.iter_stack_frames(
start_frame=inspect.currentframe(), skip_top_modules=skip_modules, config=self.config
)
),
frames_processing_func=lambda frames: self._get_stack_info_for_trace(
frames,
library_frame_context_lines=self.config.source_lines_span_library_frames,
in_app_frame_context_lines=self.config.source_lines_span_app_frames,
with_locals=self.config.collect_local_variables in ("all", "transactions"),
locals_processor_func=lambda local_var: varmap(
lambda k, v: shorten(
v,
list_length=self.config.local_var_list_max_length,
string_length=self.config.local_var_max_length,
dict_length=self.config.local_var_dict_max_length,
),
local_var,
),
),
queue_func=self.queue,
config=self.config,
agent=self,
)
self.include_paths_re = stacks.get_path_regex(self.config.include_paths) if self.config.include_paths else None
self.exclude_paths_re = stacks.get_path_regex(self.config.exclude_paths) if self.config.exclude_paths else None
self._metrics = MetricsRegistry(
self.config.metrics_interval / 1000.0, self.queue, ignore_patterns=self.config.disable_metrics
)
for path in self.config.metrics_sets:
self._metrics.register(path)
if self.config.breakdown_metrics:
self._metrics.register("elasticapm.metrics.sets.breakdown.BreakdownMetricSet")
compat.atexit_register(self.close)
if self.config.central_config:
self._config_updater = IntervalTimer(
update_config, 1, "eapm conf updater", daemon=True, args=(self,), evaluate_function_interval=True
)
self._config_updater.start()
else:
self._config_updater = None
def get_handler(self, name):
return import_string(name)
def capture(self, event_type, date=None, context=None, custom=None, stack=None, handled=True, **kwargs):
"""
Captures and processes an event and pipes it off to Client.send.
"""
if event_type == "Exception":
# never gather log stack for exceptions
stack = False
data = self._build_msg_for_logging(
event_type, date=date, context=context, custom=custom, stack=stack, handled=handled, **kwargs
)
if data:
# queue data, and flush the queue if this is an unhandled exception
self.queue(ERROR, data, flush=not handled)
return data["id"]
def capture_message(self, message=None, param_message=None, **kwargs):
"""
Creates an event from ``message``.
>>> client.capture_message('My event just happened!')
"""
return self.capture("Message", message=message, param_message=param_message, **kwargs)
def capture_exception(self, exc_info=None, handled=True, **kwargs):
"""
Creates an event from an exception.
>>> try:
>>> exc_info = sys.exc_info()
>>> client.capture_exception(exc_info)
>>> finally:
>>> del exc_info
If exc_info is not provided, or is set to True, then this method will
perform the ``exc_info = sys.exc_info()`` and the requisite clean-up
for you.
"""
return self.capture("Exception", exc_info=exc_info, handled=handled, **kwargs)
def queue(self, event_type, data, flush=False):
if self.config.disable_send:
return
if flush and is_master_process():
# don't flush in uWSGI master process to avoid ending up in an unpredictable threading state
flush = False
self._transport.queue(event_type, data, flush)
def begin_transaction(self, transaction_type, trace_parent=None, start=None):
"""
Register the start of a transaction on the client
:param transaction_type: type of the transaction, e.g. "request"
:param trace_parent: an optional TraceParent object for distributed tracing
:param start: override the start timestamp, mostly useful for testing
:return: the started transaction object
"""
return self.tracer.begin_transaction(transaction_type, trace_parent=trace_parent, start=start)
def end_transaction(self, name=None, result="", duration=None):
"""
End the current transaction.
:param name: optional name of the transaction
:param result: result of the transaction, e.g. "OK" or "HTTP 2xx"
:param duration: override duration, mostly useful for testing
:return: the ended transaction object
"""
transaction = self.tracer.end_transaction(result, name, duration=duration)
return transaction
def close(self):
if self._metrics:
self._metrics._stop_collect_timer()
if self._config_updater:
self._config_updater.cancel()
self._transport.close()
def get_service_info(self):
if self._service_info:
return self._service_info
language_version = platform.python_version()
if hasattr(sys, "pypy_version_info"):
runtime_version = ".".join(map(str, sys.pypy_version_info[:3]))
else:
runtime_version = language_version
result = {
"name": keyword_field(self.config.service_name),
"environment": keyword_field(self.config.environment),
"version": keyword_field(self.config.service_version),
"agent": {"name": "python", "version": elasticapm.VERSION},
"language": {"name": "python", "version": keyword_field(platform.python_version())},
"runtime": {
"name": keyword_field(platform.python_implementation()),
"version": keyword_field(runtime_version),
},
}
if self.config.framework_name:
result["framework"] = {
"name": keyword_field(self.config.framework_name),
"version": keyword_field(self.config.framework_version),
}
self._service_info = result
return result
def get_process_info(self):
return {
"pid": os.getpid(),
"ppid": os.getppid() if hasattr(os, "getppid") else None,
"argv": sys.argv,
"title": None, # Note: if we implement this, the value needs to be wrapped with keyword_field
}
def get_system_info(self):
system_data = {
"hostname": keyword_field(socket.gethostname()),
"architecture": platform.machine(),
"platform": platform.system().lower(),
}
system_data.update(cgroup.get_cgroup_container_metadata())
pod_name = os.environ.get("KUBERNETES_POD_NAME") or system_data["hostname"]
changed = False
if "kubernetes" in system_data:
k8s = system_data["kubernetes"]
k8s["pod"]["name"] = pod_name
else:
k8s = {"pod": {"name": pod_name}}
# get kubernetes metadata from environment
if "KUBERNETES_NODE_NAME" in os.environ:
k8s["node"] = {"name": os.environ["KUBERNETES_NODE_NAME"]}
changed = True
if "KUBERNETES_NAMESPACE" in os.environ:
k8s["namespace"] = os.environ["KUBERNETES_NAMESPACE"]
changed = True
if "KUBERNETES_POD_UID" in os.environ:
# this takes precedence over any value from /proc/self/cgroup
k8s["pod"]["uid"] = os.environ["KUBERNETES_POD_UID"]
changed = True
if changed:
system_data["kubernetes"] = k8s
return system_data
def _build_metadata(self):
data = {
"service": self.get_service_info(),
"process": self.get_process_info(),
"system": self.get_system_info(),
}
if self.config.global_labels:
data["labels"] = enforce_label_format(self.config.global_labels)
return data
def _build_msg_for_logging(
self, event_type, date=None, context=None, custom=None, stack=None, handled=True, **kwargs
):
"""
Captures, processes and serializes an event into a dict object
"""
transaction = execution_context.get_transaction()
span = execution_context.get_span()
if transaction:
transaction_context = deepcopy(transaction.context)
else:
transaction_context = {}
event_data = {}
if custom is None:
custom = {}
if date is not None:
warnings.warn(
"The date argument is no longer evaluated and will be removed in a future release", DeprecationWarning
)
date = time.time()
if stack is None:
stack = self.config.auto_log_stacks
if context:
transaction_context.update(context)
context = transaction_context
else:
context = transaction_context
event_data["context"] = context
if transaction and transaction.labels:
context["tags"] = deepcopy(transaction.labels)
# if '.' not in event_type:
# Assume it's a builtin
event_type = "elasticapm.events.%s" % event_type
handler = self.get_handler(event_type)
result = handler.capture(self, **kwargs)
if self._filter_exception_type(result):
return
# data (explicit) culprit takes over auto event detection
culprit = result.pop("culprit", None)
if custom.get("culprit"):
culprit = custom.pop("culprit")
for k, v in compat.iteritems(result):
if k not in event_data:
event_data[k] = v
log = event_data.get("log", {})
if stack and "stacktrace" not in log:
if stack is True:
frames = stacks.iter_stack_frames(skip=3, config=self.config)
else:
frames = stack
frames = stacks.get_stack_info(
frames,
with_locals=self.config.collect_local_variables in ("errors", "all"),
library_frame_context_lines=self.config.source_lines_error_library_frames,
in_app_frame_context_lines=self.config.source_lines_error_app_frames,
include_paths_re=self.include_paths_re,
exclude_paths_re=self.exclude_paths_re,
locals_processor_func=lambda local_var: varmap(
lambda k, v: shorten(
v,
list_length=self.config.local_var_list_max_length,
string_length=self.config.local_var_max_length,
dict_length=self.config.local_var_dict_max_length,
),
local_var,
),
)
log["stacktrace"] = frames
if "stacktrace" in log and not culprit:
culprit = stacks.get_culprit(log["stacktrace"], self.config.include_paths, self.config.exclude_paths)
if "level" in log and isinstance(log["level"], compat.integer_types):
log["level"] = logging.getLevelName(log["level"]).lower()
if log:
event_data["log"] = log
if culprit:
event_data["culprit"] = culprit
if "custom" in context:
context["custom"].update(custom)
else:
context["custom"] = custom
# Make sure all data is coerced
event_data = transform(event_data)
if "exception" in event_data:
event_data["exception"]["handled"] = bool(handled)
event_data["timestamp"] = int(date * 1000000)
if transaction:
if transaction.trace_parent:
event_data["trace_id"] = transaction.trace_parent.trace_id
# parent id might already be set in the handler
event_data.setdefault("parent_id", span.id if span else transaction.id)
event_data["transaction_id"] = transaction.id
event_data["transaction"] = {"sampled": transaction.is_sampled, "type": transaction.transaction_type}
return event_data
def _filter_exception_type(self, data):
exception = data.get("exception")
if not exception:
return False
exc_type = exception.get("type")
exc_module = exception.get("module")
if exc_module == "None":
exc_module = None
if exc_type in self.filter_exception_types_dict:
exc_to_filter_module = self.filter_exception_types_dict[exc_type]
if not exc_to_filter_module or exc_to_filter_module == exc_module:
if exc_module:
exc_name = "%s.%s" % (exc_module, exc_type)
else:
exc_name = exc_type
self.logger.info("Ignored %s exception due to exception type filter", exc_name)
return True
return False
def _get_stack_info_for_trace(
self,
frames,
library_frame_context_lines=None,
in_app_frame_context_lines=None,
with_locals=True,
locals_processor_func=None,
):
"""Overrideable in derived clients to add frames/info, e.g. templates"""
return stacks.get_stack_info(
frames,
library_frame_context_lines=library_frame_context_lines,
in_app_frame_context_lines=in_app_frame_context_lines,
with_locals=with_locals,
include_paths_re=self.include_paths_re,
exclude_paths_re=self.exclude_paths_re,
locals_processor_func=locals_processor_func,
)
def load_processors(self):
"""
Loads processors from self.config.processors, as well as constants.HARDCODED_PROCESSORS.
Duplicate processors (based on the path) will be discarded.
:return: a list of callables
"""
processors = itertools.chain(self.config.processors, constants.HARDCODED_PROCESSORS)
seen = {}
# setdefault has the nice property that it returns the value that it just set on the dict
return [seen.setdefault(path, import_string(path)) for path in processors if path not in seen]
def __init__(self, config=None, **inline):
# configure loggers first
cls = self.__class__
self.logger = logging.getLogger("%s.%s" %
(cls.__module__, cls.__name__))
self.error_logger = logging.getLogger("elasticapm.errors")
self.tracer = None
self.processors = []
self.filter_exception_types_dict = {}
self._service_info = None
self.config = Config(config, inline_dict=inline)
if self.config.errors:
for msg in self.config.errors.values():
self.error_logger.error(msg)
self.config.disable_send = True
headers = {
"Content-Type": "application/x-ndjson",
"Content-Encoding": "gzip",
"User-Agent": "elasticapm-python/%s" % elasticapm.VERSION,
}
if self.config.secret_token:
headers["Authorization"] = "Bearer %s" % self.config.secret_token
transport_kwargs = {
"metadata": self._build_metadata(),
"headers": headers,
"verify_server_cert": self.config.verify_server_cert,
"timeout": self.config.server_timeout,
"max_flush_time": self.config.api_request_time / 1000.0,
"max_buffer_size": self.config.api_request_size,
}
self._api_endpoint_url = compat.urlparse.urljoin(
self.config.server_url if self.config.server_url.endswith("/") else
self.config.server_url + "/",
constants.EVENTS_API_PATH,
)
self._transport = import_string(self.config.transport_class)(
self._api_endpoint_url, **transport_kwargs)
for exc_to_filter in self.config.filter_exception_types or []:
exc_to_filter_type = exc_to_filter.split(".")[-1]
exc_to_filter_module = ".".join(exc_to_filter.split(".")[:-1])
self.filter_exception_types_dict[
exc_to_filter_type] = exc_to_filter_module
self.processors = [import_string(p) for p in self.config.processors
] if self.config.processors else []
if platform.python_implementation() == "PyPy":
# PyPy introduces a `_functools.partial.__call__` frame due to our use
# of `partial` in AbstractInstrumentedModule
skip_modules = ("elasticapm.", "_functools")
else:
skip_modules = ("elasticapm.", )
self.tracer = Tracer(
frames_collector_func=lambda: list(
stacks.iter_stack_frames(start_frame=inspect.currentframe(),
skip_top_modules=skip_modules)),
frames_processing_func=lambda frames: self.
_get_stack_info_for_trace(
frames,
library_frame_context_lines=self.config.
source_lines_span_library_frames,
in_app_frame_context_lines=self.config.
source_lines_span_app_frames,
with_locals=self.config.collect_local_variables in
("all", "transactions"),
locals_processor_func=lambda local_var: varmap(
lambda k, v: shorten(
v,
list_length=self.config.local_var_list_max_length,
string_length=self.config.local_var_max_length,
),
local_var,
),
),
queue_func=self.queue,
sample_rate=self.config.transaction_sample_rate,
max_spans=self.config.transaction_max_spans,
span_frames_min_duration=self.config.span_frames_min_duration,
ignore_patterns=self.config.transactions_ignore_patterns,
)
self.include_paths_re = stacks.get_path_regex(
self.config.include_paths) if self.config.include_paths else None
self.exclude_paths_re = stacks.get_path_regex(
self.config.exclude_paths) if self.config.exclude_paths else None
self._metrics = MetricsRegistry(self.config.metrics_interval / 1000.0,
self.queue)
for path in self.config.metrics_sets:
self._metrics.register(path)
compat.atexit_register(self.close)
class Client(object):
"""
The base ElasticAPM client, which handles communication over the
HTTP API to the APM Server.
Will read default configuration from the environment variable
``ELASTIC_APM_APP_NAME`` and ``ELASTIC_APM_SECRET_TOKEN``
if available. ::
>>> from elasticapm import Client
>>> # Read configuration from environment
>>> client = Client()
>>> # Configure the client manually
>>> client = Client(
>>> include_paths=['my.package'],
>>> service_name='myapp',
>>> secret_token='secret_token',
>>> )
>>> # Record an exception
>>> try:
>>> 1/0
>>> except ZeroDivisionError:
>>> ident = client.capture_exception()
>>> print ("Exception caught; reference is %%s" %% ident)
"""
logger = get_logger("elasticapm")
def __init__(self, config=None, **inline):
# configure loggers first
cls = self.__class__
self.logger = get_logger("%s.%s" % (cls.__module__, cls.__name__))
self.error_logger = get_logger("elasticapm.errors")
self._pid = None
self._thread_starter_lock = threading.Lock()
self._thread_managers = {}
self.tracer = None
self.processors = []
self.filter_exception_types_dict = {}
self._service_info = None
# setting server_version here is mainly used for testing
self.server_version = inline.pop("server_version", None)
self.check_python_version()
config = Config(config, inline_dict=inline)
if config.errors:
for msg in config.errors.values():
self.error_logger.error(msg)
config.disable_send = True
if config.service_name == "python_service":
self.logger.warning(
"No custom SERVICE_NAME was set -- using non-descript default 'python_service'"
)
self.config = VersionedConfig(config, version=None)
# Insert the log_record_factory into the logging library
# The LogRecordFactory functionality is only available on python 3.2+
if compat.PY3 and not self.config.disable_log_record_factory:
record_factory = logging.getLogRecordFactory()
# Only way to know if it's wrapped is to create a log record
throwaway_record = record_factory(__name__, logging.DEBUG,
__file__, 252, "dummy_msg", [],
None)
if not hasattr(throwaway_record, "elasticapm_labels"):
self.logger.debug(
"Inserting elasticapm log_record_factory into logging")
# Late import due to circular imports
import elasticapm.handlers.logging as elastic_logging
new_factory = elastic_logging.log_record_factory(
record_factory)
logging.setLogRecordFactory(new_factory)
headers = {
"Content-Type": "application/x-ndjson",
"Content-Encoding": "gzip",
"User-Agent": self.get_user_agent(),
}
transport_kwargs = {
"headers": headers,
"verify_server_cert": self.config.verify_server_cert,
"server_cert": self.config.server_cert,
"timeout": self.config.server_timeout,
"processors": self.load_processors(),
}
self._api_endpoint_url = compat.urlparse.urljoin(
self.config.server_url if self.config.server_url.endswith("/") else
self.config.server_url + "/",
constants.EVENTS_API_PATH,
)
transport_class = import_string(self.config.transport_class)
self._transport = transport_class(url=self._api_endpoint_url,
client=self,
**transport_kwargs)
self.config.transport = self._transport
self._thread_managers["transport"] = self._transport
for exc_to_filter in self.config.filter_exception_types or []:
exc_to_filter_type = exc_to_filter.split(".")[-1]
exc_to_filter_module = ".".join(exc_to_filter.split(".")[:-1])
self.filter_exception_types_dict[
exc_to_filter_type] = exc_to_filter_module
if platform.python_implementation() == "PyPy":
# PyPy introduces a `_functools.partial.__call__` frame due to our use
# of `partial` in AbstractInstrumentedModule
skip_modules = ("elasticapm.", "_functools")
else:
skip_modules = ("elasticapm.", )
self.tracer = Tracer(
frames_collector_func=lambda: list(
stacks.iter_stack_frames(start_frame=inspect.currentframe(),
skip_top_modules=skip_modules,
config=self.config)),
frames_processing_func=lambda frames: self.
_get_stack_info_for_trace(
frames,
library_frame_context_lines=self.config.
source_lines_span_library_frames,
in_app_frame_context_lines=self.config.
source_lines_span_app_frames,
with_locals=self.config.collect_local_variables in
("all", "transactions"),
locals_processor_func=lambda local_var: varmap(
lambda k, v: shorten(
v,
list_length=self.config.local_var_list_max_length,
string_length=self.config.local_var_max_length,
dict_length=self.config.local_var_dict_max_length,
),
local_var,
),
),
queue_func=self.queue,
config=self.config,
agent=self,
)
self.include_paths_re = stacks.get_path_regex(
self.config.include_paths) if self.config.include_paths else None
self.exclude_paths_re = stacks.get_path_regex(
self.config.exclude_paths) if self.config.exclude_paths else None
self._metrics = MetricsRegistry(self)
for path in self.config.metrics_sets:
self._metrics.register(path)
if self.config.breakdown_metrics:
self._metrics.register(
"elasticapm.metrics.sets.breakdown.BreakdownMetricSet")
if self.config.prometheus_metrics:
self._metrics.register(
"elasticapm.metrics.sets.prometheus.PrometheusMetrics")
if self.config.metrics_interval:
self._thread_managers["metrics"] = self._metrics
compat.atexit_register(self.close)
if self.config.central_config:
self._thread_managers["config"] = self.config
else:
self._config_updater = None
if self.config.use_elastic_excepthook:
self.original_excepthook = sys.excepthook
sys.excepthook = self._excepthook
if config.enabled:
self.start_threads()
# Save this Client object as the global CLIENT_SINGLETON
set_client(self)
def start_threads(self):
current_pid = os.getpid()
if self._pid != current_pid:
with self._thread_starter_lock:
self.logger.debug(
"Detected PID change from %r to %r, starting threads",
self._pid, current_pid)
for manager_type, manager in sorted(
self._thread_managers.items(),
key=lambda item: item[1].start_stop_order):
self.logger.debug("Starting %s thread", manager_type)
manager.start_thread(pid=current_pid)
self._pid = current_pid
def get_handler(self, name):
return import_string(name)
def capture(self,
event_type,
date=None,
context=None,
custom=None,
stack=None,
handled=True,
**kwargs):
"""
Captures and processes an event and pipes it off to Client.send.
"""
if not self.config.is_recording:
return
if event_type == "Exception":
# never gather log stack for exceptions
stack = False
data = self._build_msg_for_logging(event_type,
date=date,
context=context,
custom=custom,
stack=stack,
handled=handled,
**kwargs)
if data:
# queue data, and flush the queue if this is an unhandled exception
self.queue(ERROR, data, flush=not handled)
return data["id"]
def capture_message(self, message=None, param_message=None, **kwargs):
"""
Creates an event from ``message``.
>>> client.capture_message('My event just happened!')
"""
return self.capture("Message",
message=message,
param_message=param_message,
**kwargs)
def capture_exception(self, exc_info=None, handled=True, **kwargs):
"""
Creates an event from an exception.
>>> try:
>>> exc_info = sys.exc_info()
>>> client.capture_exception(exc_info)
>>> finally:
>>> del exc_info
If exc_info is not provided, or is set to True, then this method will
perform the ``exc_info = sys.exc_info()`` and the requisite clean-up
for you.
"""
return self.capture("Exception",
exc_info=exc_info,
handled=handled,
**kwargs)
def queue(self, event_type, data, flush=False):
if self.config.disable_send:
return
self.start_threads()
if flush and is_master_process():
# don't flush in uWSGI master process to avoid ending up in an unpredictable threading state
flush = False
self._transport.queue(event_type, data, flush)
def begin_transaction(self,
transaction_type,
trace_parent=None,
start=None):
"""
Register the start of a transaction on the client
:param transaction_type: type of the transaction, e.g. "request"
:param trace_parent: an optional TraceParent object for distributed tracing
:param start: override the start timestamp, mostly useful for testing
:return: the started transaction object
"""
if self.config.is_recording:
return self.tracer.begin_transaction(transaction_type,
trace_parent=trace_parent,
start=start)
def end_transaction(self, name=None, result="", duration=None):
"""
End the current transaction.
:param name: optional name of the transaction
:param result: result of the transaction, e.g. "OK" or "HTTP 2xx"
:param duration: override duration, mostly useful for testing
:return: the ended transaction object
"""
transaction = self.tracer.end_transaction(result,
name,
duration=duration)
return transaction
def close(self):
if self.config.enabled:
with self._thread_starter_lock:
for _, manager in sorted(
self._thread_managers.items(),
key=lambda item: item[1].start_stop_order):
manager.stop_thread()
global CLIENT_SINGLETON
CLIENT_SINGLETON = None
def get_service_info(self):
if self._service_info:
return self._service_info
language_version = platform.python_version()
if hasattr(sys, "pypy_version_info"):
runtime_version = ".".join(map(str, sys.pypy_version_info[:3]))
else:
runtime_version = language_version
result = {
"name": keyword_field(self.config.service_name),
"environment": keyword_field(self.config.environment),
"version": keyword_field(self.config.service_version),
"agent": {
"name": "python",
"version": elasticapm.VERSION
},
"language": {
"name": "python",
"version": keyword_field(platform.python_version())
},
"runtime": {
"name": keyword_field(platform.python_implementation()),
"version": keyword_field(runtime_version),
},
}
if self.config.framework_name:
result["framework"] = {
"name": keyword_field(self.config.framework_name),
"version": keyword_field(self.config.framework_version),
}
if self.config.service_node_name:
result["node"] = {
"configured_name": keyword_field(self.config.service_node_name)
}
self._service_info = result
return result
def get_process_info(self):
return {
"pid": os.getpid(),
"ppid": os.getppid() if hasattr(os, "getppid") else None,
"argv": sys.argv,
"title":
None, # Note: if we implement this, the value needs to be wrapped with keyword_field
}
def get_system_info(self):
system_data = {
"hostname": keyword_field(self.config.hostname),
"architecture": platform.machine(),
"platform": platform.system().lower(),
}
system_data.update(cgroup.get_cgroup_container_metadata())
pod_name = os.environ.get(
"KUBERNETES_POD_NAME") or system_data["hostname"]
changed = False
if "kubernetes" in system_data:
k8s = system_data["kubernetes"]
k8s["pod"]["name"] = pod_name
else:
k8s = {"pod": {"name": pod_name}}
# get kubernetes metadata from environment
if "KUBERNETES_NODE_NAME" in os.environ:
k8s["node"] = {"name": os.environ["KUBERNETES_NODE_NAME"]}
changed = True
if "KUBERNETES_NAMESPACE" in os.environ:
k8s["namespace"] = os.environ["KUBERNETES_NAMESPACE"]
changed = True
if "KUBERNETES_POD_UID" in os.environ:
# this takes precedence over any value from /proc/self/cgroup
k8s["pod"]["uid"] = os.environ["KUBERNETES_POD_UID"]
changed = True
if changed:
system_data["kubernetes"] = k8s
return system_data
def get_cloud_info(self):
"""
Detects if the app is running in a cloud provider and fetches relevant
metadata from the cloud provider's metadata endpoint.
"""
provider = str(self.config.cloud_provider).lower()
if not provider or provider == "none" or provider == "false":
return {}
if provider == "aws":
data = cloud.aws_metadata()
if not data:
self.logger.warning(
"Cloud provider {0} defined, but no metadata was found.".
format(provider))
return data
elif provider == "gcp":
data = cloud.gcp_metadata()
if not data:
self.logger.warning(
"Cloud provider {0} defined, but no metadata was found.".
format(provider))
return data
elif provider == "azure":
data = cloud.azure_metadata()
if not data:
self.logger.warning(
"Cloud provider {0} defined, but no metadata was found.".
format(provider))
return data
elif provider == "auto" or provider == "true":
# Trial and error
data = {}
data = cloud.aws_metadata()
if data:
return data
data = cloud.gcp_metadata()
if data:
return data
data = cloud.azure_metadata()
return data
else:
self.logger.warning(
"Unknown value for CLOUD_PROVIDER, skipping cloud metadata: {}"
.format(provider))
return {}
def get_user_agent(self) -> str:
"""
Compiles the user agent, which will be added as a header to all requests
to the APM Server
"""
if self.config.service_version:
service_version = re.sub(
r"[^\t _\x21-\x27\x2a-\x5b\x5d-\x7e\x80-\xff]", "_",
self.config.service_version)
return "apm-agent-python/{} ({} {})".format(
elasticapm.VERSION, self.config.service_name, service_version)
else:
return "apm-agent-python/{} ({})".format(elasticapm.VERSION,
self.config.service_name)
def build_metadata(self):
data = {
"service": self.get_service_info(),
"process": self.get_process_info(),
"system": self.get_system_info(),
"cloud": self.get_cloud_info(),
}
if not data["cloud"]:
data.pop("cloud")
if self.config.global_labels:
data["labels"] = enforce_label_format(self.config.global_labels)
return data
def _build_msg_for_logging(self,
event_type,
date=None,
context=None,
custom=None,
stack=None,
handled=True,
**kwargs):
"""
Captures, processes and serializes an event into a dict object
"""
transaction = execution_context.get_transaction()
span = execution_context.get_span()
if transaction:
transaction_context = deepcopy(transaction.context)
else:
transaction_context = {}
event_data = {}
if custom is None:
custom = {}
if date is not None:
warnings.warn(
"The date argument is no longer evaluated and will be removed in a future release",
DeprecationWarning)
date = time.time()
if stack is None:
stack = self.config.auto_log_stacks
if context:
transaction_context.update(context)
context = transaction_context
else:
context = transaction_context
event_data["context"] = context
if transaction and transaction.labels:
context["tags"] = deepcopy(transaction.labels)
# if '.' not in event_type:
# Assume it's a builtin
event_type = "elasticapm.events.%s" % event_type
handler = self.get_handler(event_type)
result = handler.capture(self, **kwargs)
if self._filter_exception_type(result):
return
# data (explicit) culprit takes over auto event detection
culprit = result.pop("culprit", None)
if custom.get("culprit"):
culprit = custom.pop("culprit")
for k, v in compat.iteritems(result):
if k not in event_data:
event_data[k] = v
log = event_data.get("log", {})
if stack and "stacktrace" not in log:
if stack is True:
frames = stacks.iter_stack_frames(skip=3, config=self.config)
else:
frames = stack
frames = stacks.get_stack_info(
frames,
with_locals=self.config.collect_local_variables
in ("errors", "all"),
library_frame_context_lines=self.config.
source_lines_error_library_frames,
in_app_frame_context_lines=self.config.
source_lines_error_app_frames,
include_paths_re=self.include_paths_re,
exclude_paths_re=self.exclude_paths_re,
locals_processor_func=lambda local_var: varmap(
lambda k, v: shorten(
v,
list_length=self.config.local_var_list_max_length,
string_length=self.config.local_var_max_length,
dict_length=self.config.local_var_dict_max_length,
),
local_var,
),
)
log["stacktrace"] = frames
if "stacktrace" in log and not culprit:
culprit = stacks.get_culprit(log["stacktrace"],
self.config.include_paths,
self.config.exclude_paths)
if "level" in log and isinstance(log["level"], compat.integer_types):
log["level"] = logging.getLevelName(log["level"]).lower()
if log:
event_data["log"] = log
if culprit:
event_data["culprit"] = culprit
if "custom" in context:
context["custom"].update(custom)
else:
context["custom"] = custom
# Make sure all data is coerced
event_data = transform(event_data)
if "exception" in event_data:
event_data["exception"]["handled"] = bool(handled)
event_data["timestamp"] = int(date * 1000000)
if transaction:
if transaction.trace_parent:
event_data["trace_id"] = transaction.trace_parent.trace_id
# parent id might already be set in the handler
event_data.setdefault("parent_id",
span.id if span else transaction.id)
event_data["transaction_id"] = transaction.id
event_data["transaction"] = {
"sampled": transaction.is_sampled,
"type": transaction.transaction_type
}
return event_data
def _filter_exception_type(self, data):
exception = data.get("exception")
if not exception:
return False
exc_type = exception.get("type")
exc_module = exception.get("module")
if exc_module == "None":
exc_module = None
if exc_type in self.filter_exception_types_dict:
exc_to_filter_module = self.filter_exception_types_dict[exc_type]
if not exc_to_filter_module or exc_to_filter_module == exc_module:
if exc_module:
exc_name = "%s.%s" % (exc_module, exc_type)
else:
exc_name = exc_type
self.logger.debug(
"Ignored %s exception due to exception type filter",
exc_name)
return True
return False
def _get_stack_info_for_trace(
self,
frames,
library_frame_context_lines=None,
in_app_frame_context_lines=None,
with_locals=True,
locals_processor_func=None,
):
"""Overrideable in derived clients to add frames/info, e.g. templates"""
return stacks.get_stack_info(
frames,
library_frame_context_lines=library_frame_context_lines,
in_app_frame_context_lines=in_app_frame_context_lines,
with_locals=with_locals,
include_paths_re=self.include_paths_re,
exclude_paths_re=self.exclude_paths_re,
locals_processor_func=locals_processor_func,
)
def _excepthook(self, type_, value, traceback):
try:
self.original_excepthook(type_, value, traceback)
except Exception:
self.capture_exception(handled=False)
finally:
self.capture_exception(exc_info=(type_, value, traceback),
handled=False)
def load_processors(self):
"""
Loads processors from self.config.processors, as well as constants.HARDCODED_PROCESSORS.
Duplicate processors (based on the path) will be discarded.
:return: a list of callables
"""
processors = itertools.chain(self.config.processors,
constants.HARDCODED_PROCESSORS)
seen = {}
# setdefault has the nice property that it returns the value that it just set on the dict
return [
seen.setdefault(path, import_string(path)) for path in processors
if path not in seen
]
def should_ignore_url(self, url):
if self.config.transaction_ignore_urls:
for pattern in self.config.transaction_ignore_urls:
if pattern.match(url):
return True
return False
def check_python_version(self):
v = tuple(map(int, platform.python_version_tuple()[:2]))
if v == (2, 7):
warnings.warn(
("The Elastic APM agent will stop supporting Python 2.7 starting in 6.0.0 -- "
"Please upgrade to Python 3.5+ to continue to use the latest features."
),
PendingDeprecationWarning,
)
elif v < (3, 5):
warnings.warn("The Elastic APM agent only supports Python 3.5+",
DeprecationWarning)
def check_server_version(self,
gte: Optional[Tuple[int]] = None,
lte: Optional[Tuple[int]] = None) -> bool:
"""
Check APM Server version against greater-or-equal and/or lower-or-equal limits, provided as tuples of integers.
If server_version is not set, always returns True.
:param gte: a tuple of ints describing the greater-or-equal limit, e.g. (7, 16)
:param lte: a tuple of ints describing the lower-or-equal limit, e.g. (7, 99)
:return: bool
"""
if not self.server_version:
return True
gte = gte or (0, )
lte = lte or (
2**32,
) # let's assume APM Server version will never be greater than 2^32
return bool(gte <= self.server_version <= lte)
def __init__(self, config=None, **inline):
# configure loggers first
cls = self.__class__
self.logger = get_logger("%s.%s" % (cls.__module__, cls.__name__))
self.error_logger = get_logger("elasticapm.errors")
self._pid = None
self._thread_starter_lock = threading.Lock()
self._thread_managers = {}
self.tracer = None
self.processors = []
self.filter_exception_types_dict = {}
self._service_info = None
self.check_python_version()
config = Config(config, inline_dict=inline)
if config.errors:
for msg in config.errors.values():
self.error_logger.error(msg)
config.disable_send = True
if config.service_name == "python_service":
self.logger.warning(
"No custom SERVICE_NAME was set -- using non-descript default 'python_service'"
)
self.config = VersionedConfig(config, version=None)
# Insert the log_record_factory into the logging library
# The LogRecordFactory functionality is only available on python 3.2+
if compat.PY3 and not self.config.disable_log_record_factory:
record_factory = logging.getLogRecordFactory()
# Only way to know if it's wrapped is to create a log record
throwaway_record = record_factory(__name__, logging.DEBUG,
__file__, 252, "dummy_msg", [],
None)
if not hasattr(throwaway_record, "elasticapm_labels"):
self.logger.debug(
"Inserting elasticapm log_record_factory into logging")
# Late import due to circular imports
import elasticapm.handlers.logging as elastic_logging
new_factory = elastic_logging.log_record_factory(
record_factory)
logging.setLogRecordFactory(new_factory)
headers = {
"Content-Type": "application/x-ndjson",
"Content-Encoding": "gzip",
"User-Agent": "elasticapm-python/%s" % elasticapm.VERSION,
}
transport_kwargs = {
"headers": headers,
"verify_server_cert": self.config.verify_server_cert,
"server_cert": self.config.server_cert,
"timeout": self.config.server_timeout,
"processors": self.load_processors(),
}
self._api_endpoint_url = compat.urlparse.urljoin(
self.config.server_url if self.config.server_url.endswith("/") else
self.config.server_url + "/",
constants.EVENTS_API_PATH,
)
transport_class = import_string(self.config.transport_class)
self._transport = transport_class(self._api_endpoint_url, self,
**transport_kwargs)
self.config.transport = self._transport
self._thread_managers["transport"] = self._transport
for exc_to_filter in self.config.filter_exception_types or []:
exc_to_filter_type = exc_to_filter.split(".")[-1]
exc_to_filter_module = ".".join(exc_to_filter.split(".")[:-1])
self.filter_exception_types_dict[
exc_to_filter_type] = exc_to_filter_module
if platform.python_implementation() == "PyPy":
# PyPy introduces a `_functools.partial.__call__` frame due to our use
# of `partial` in AbstractInstrumentedModule
skip_modules = ("elasticapm.", "_functools")
else:
skip_modules = ("elasticapm.", )
self.tracer = Tracer(
frames_collector_func=lambda: list(
stacks.iter_stack_frames(start_frame=inspect.currentframe(),
skip_top_modules=skip_modules,
config=self.config)),
frames_processing_func=lambda frames: self.
_get_stack_info_for_trace(
frames,
library_frame_context_lines=self.config.
source_lines_span_library_frames,
in_app_frame_context_lines=self.config.
source_lines_span_app_frames,
with_locals=self.config.collect_local_variables in
("all", "transactions"),
locals_processor_func=lambda local_var: varmap(
lambda k, v: shorten(
v,
list_length=self.config.local_var_list_max_length,
string_length=self.config.local_var_max_length,
dict_length=self.config.local_var_dict_max_length,
),
local_var,
),
),
queue_func=self.queue,
config=self.config,
agent=self,
)
self.include_paths_re = stacks.get_path_regex(
self.config.include_paths) if self.config.include_paths else None
self.exclude_paths_re = stacks.get_path_regex(
self.config.exclude_paths) if self.config.exclude_paths else None
self._metrics = MetricsRegistry(self)
for path in self.config.metrics_sets:
self._metrics.register(path)
if self.config.breakdown_metrics:
self._metrics.register(
"elasticapm.metrics.sets.breakdown.BreakdownMetricSet")
if self.config.prometheus_metrics:
self._metrics.register(
"elasticapm.metrics.sets.prometheus.PrometheusMetrics")
self._thread_managers["metrics"] = self._metrics
compat.atexit_register(self.close)
if self.config.central_config:
self._thread_managers["config"] = self.config
else:
self._config_updater = None
if self.config.use_elastic_excepthook:
self.original_excepthook = sys.excepthook
sys.excepthook = self._excepthook
if config.enabled:
self.start_threads()
# Save this Client object as the global CLIENT_SINGLETON
set_client(self)
def test_metrics_registry(elasticapm_client):
registry = MetricsRegistry(elasticapm_client)
registry.register("tests.metrics.base_tests.DummyMetricSet")
registry.collect()
assert len(elasticapm_client.events[constants.METRICSET])
