def _track_request_duration(self, observer: Observer) -> None:
""" Track Request execution time
Calculated by getting the time it takes to make an incoming request
and dividing over the amount of incoming requests over an elapsed time.
"""
last_average_duration = requests_map.get("last_average_duration", 0)
interval_duration = (
self._span_processor.request_duration
- requests_map.get("last_duration", 0)
)
interval_count = self._span_processor.request_count - requests_map.get(
"last_count", 0
)
try:
result = interval_duration / interval_count
requests_map["last_count"] = self._span_processor.request_count
requests_map["last_average_duration"] = result
requests_map[
"last_duration"
] = self._span_processor.request_duration
observer.observe(int(result), self._labels)
except ZeroDivisionError:
# If interval_count is 0, exporter call made too close to previous
# Return the previous result if this is the case
observer.observe(int(last_average_duration), self._labels)
def _track_dependency_rate(self, observer: Observer) -> None:
"""Track Dependency rate
Calculated by obtaining the number of outgoing requests made
using the requests library within an elapsed time and dividing
that value over the elapsed time.
"""
current_count = dependency_map.get("count", 0)
current_time = time.time()
last_count = dependency_map.get("last_count", 0)
last_time = dependency_map.get("last_time")
last_result = dependency_map.get("last_result", 0.0)
try:
# last_time is None the very first time this function is called
if last_time is not None:
elapsed_seconds = current_time - last_time
interval_count = current_count - last_count
result = interval_count / elapsed_seconds
else:
result = 0.0
dependency_map["last_time"] = current_time
dependency_map["last_count"] = current_count
dependency_map["last_result"] = result
observer.observe(result, self._labels)
except ZeroDivisionError:
# If elapsed_seconds is 0, exporter call made too close to previous
# Return the previous result if this is the case
observer.observe(last_result, self._labels)
def _track_request_failed_rate(self, observer: Observer) -> None:
""" Track Request failed execution rate
Calculated by obtaining by getting the number of failed incoming requests
made to an HTTPServer within an elapsed time and dividing that value
over the elapsed time.
"""
current_time = time.time()
last_rate = requests_map.get("last_rate", 0.0)
last_time = requests_map.get("last_time")
try:
# last_rate_time is None the first time this function is called
if last_time is not None:
interval_time = current_time - requests_map.get("last_time", 0)
interval_count = requests_map.get("failed_count",
0) - requests_map.get(
"last_failed_count", 0)
result = interval_count / interval_time
else:
result = 0.0
requests_map["last_time"] = current_time
requests_map["last_failed_count"] = requests_map.get(
"failed_count", 0)
requests_map["last_rate"] = result
observer.observe(result, self._labels)
except ZeroDivisionError:
# If elapsed_seconds is 0, exporter call made too close to previous
# Return the previous result if this is the case
observer.observe(last_rate, self._labels)
def _track_memory(self, observer: Observer) -> None:
""" Track Memory
Available memory is defined as memory that can be given instantly to
processes without the system going into swap.
"""
observer.observe(psutil.virtual_memory().available, self._labels)
def _track_commited_memory(self, observer: Observer) -> None:
""" Track Commited Memory
Available commited memory is defined as total memory minus available memory.
"""
observer.observe(
psutil.virtual_memory().total - psutil.virtual_memory().available,
self._labels,
)
def _track_cpu(self, observer: Observer) -> None:
""" Track CPU time
Processor time is defined as a float representing the current system
wide CPU utilization minus idle CPU time as a percentage. Idle CPU
time is defined as the time spent doing nothing. Return values range
from 0.0 to 100.0 inclusive.
"""
cpu_times_percent = psutil.cpu_times_percent()
observer.observe(100.0 - cpu_times_percent.idle, self._labels)
def _track_process_memory(self, observer: Observer) -> None:
""" Track Memory
Available memory is defined as memory that can be given instantly to
processes without the system going into swap.
"""
try:
observer.observe(PROCESS.memory_info().rss, self._labels)
except Exception: # pylint: disable=broad-except
logger.warning("Error handling get process private bytes.")
def _get_runtime_gc_count(self, observer: metrics.Observer) -> None:
"""Observer callback for garbage collection
Args:
observer: the observer to update
"""
gc_count = gc.get_count()
for index, count in enumerate(gc_count):
self._runtime_gc_labels["count"] = str(index)
observer.observe(count, self._runtime_gc_labels)
def _get_runtime_cpu(self, observer: metrics.Observer) -> None:
"""Observer callback for runtime CPU
Args:
observer: the observer to update
"""
proc_cpu = self._proc.cpu_times()
for _type in self._config["runtime_cpu"]:
self._runtime_cpu_labels["type"] = _type
observer.observe(
getattr(proc_cpu, _type), self._runtime_cpu_labels
)
def _get_runtime_memory(self, observer: metrics.Observer) -> None:
"""Observer callback for runtime memory
Args:
observer: the observer to update
"""
proc_memory = self._proc.memory_info()
for _type in self._config["runtime_memory"]:
self._runtime_memory_labels["type"] = _type
observer.observe(
getattr(proc_memory, _type), self._runtime_memory_labels
)
def _get_network_bytes(self, observer: metrics.Observer) -> None:
"""Observer callback for network bytes
Args:
observer: the observer to update
"""
net_io = psutil.net_io_counters()
for _type in self._config["network_bytes"]:
self._network_bytes_labels["type"] = _type
observer.observe(
getattr(net_io, _type), self._network_bytes_labels
)
def _get_system_cpu(self, observer: metrics.Observer) -> None:
"""Observer callback for system cpu
Args:
observer: the observer to update
"""
cpu_times = psutil.cpu_times()
for _type in self._config["system_cpu"]:
self._system_cpu_labels["type"] = _type
observer.observe(
getattr(cpu_times, _type), self._system_cpu_labels
)
def _get_system_memory(self, observer: metrics.Observer) -> None:
"""Observer callback for memory available
Args:
observer: the observer to update
"""
system_memory = psutil.virtual_memory()
for metric in self._config["system_memory"]:
self._system_memory_labels["type"] = metric
observer.observe(
getattr(system_memory, metric), self._system_memory_labels
)
def _track_process_cpu(self, observer: Observer) -> None:
""" Track Process CPU time
Returns a derived gauge for the CPU usage for the current process.
Return values range from 0.0 to 100.0 inclusive.
"""
try:
# In the case of a process running on multiple threads on different
# CPU cores, the returned value of cpu_percent() can be > 100.0. We
# normalize the cpu process using the number of logical CPUs
cpu_count = psutil.cpu_count(logical=True)
observer.observe(PROCESS.cpu_percent() / cpu_count, self._labels)
except Exception: # pylint: disable=broad-except
logger.warning("Error handling get process cpu usage.")
def _track_dependency_duration(self, observer: Observer) -> None:
"""Track Dependency average duration
Calculated by getting the time it takes to make an outgoing request
and dividing over the amount of outgoing requests over an elapsed time.
"""
last_average_duration = dependency_map.get("last_average_duration",
0.0)
interval_duration = dependency_map.get(
"duration", 0.0) - dependency_map.get("last_duration", 0.0)
interval_count = dependency_map.get("count", 0) - dependency_map.get(
"last_count", 0)
try:
result = interval_duration / interval_count
dependency_map["last_count"] = dependency_map.get("count", 0)
dependency_map["last_average_duration"] = result
dependency_map["last_duration"] = dependency_map.get(
"duration", 0.0)
observer.observe(result, self._labels)
except ZeroDivisionError:
# If interval_count is 0, exporter call made too close to previous
# Return the previous result if this is the case
observer.observe(last_average_duration, self._labels)