def test_gauge_set_invalid_value(Gauge):
"""Try to set a gauge to an invalid value"""
g = Gauge(
"test_gauge_set_invalid_value",
"test_gauge_set_invalid_value",
)
with pytest.raises(ValueError):
g.set("badstring")
def test_gauge_multiproc_override_invalid(Gauge):
"""Test overriding the default gauge multiproc mode with something
invalid"""
new_multiprocess_mode = "foo"
with pytest.raises(ValueError, match="Invalid multiprocess mode"):
_ = Gauge(
"test_gauge_multiproc_override_invalid",
"test_gauge_multiproc_override_invalid",
multiprocess_mode=new_multiprocess_mode,
)
def test_gauge_set_multiproc_mode(Gauge, test_params):
"""
Test setting the gauge multiproc mode
test_params is a list of dicts that support three keys:
'metric_args': list of arguments to pass to Gauge
'metric_kwargs': dict of keyword arguments to pass to Gauge
'wanted_multiprocess_mode': Intended multiprocess mode of Gauge after
instantiation
"""
g = Gauge(*test_params["metric_args"],
**test_params.get("metric_kwargs", {}))
assert g._multiprocess_mode == test_params.get(
"wanted_multiprocess_mode", pytest.gauge_default_multiproc_mode)
# DecisionEngine metrics
STATUS_HISTOGRAM = Histogram("de_client_status_duration_seconds",
"Time to run de-client --status")
PRINT_PRODUCT_HISTOGRAM = Histogram("de_client_print_product_duration_seconds",
"Time to run de-client --print-product")
START_CHANNEL_HISTOGRAM = Histogram("de_client_start_channel_duration_seconds",
"Time to run de-client --start-channel",
["channel_name"])
RM_CHANNEL_HISTOGRAM = Histogram("de_client_rm_channel_duration_seconds",
"Time to run de-client --stop-channel",
["channel_name"])
QUERY_TOOL_HISTOGRAM = Histogram("de_client_query_duration_seconds",
"Time to run de-client --query", ["product"])
METRICS_HISTOGRAM = Histogram("de_client_metrics_duration_seconds",
"Time to run de-client --status")
WORKERS_COUNT = Gauge("de_workers_total",
"Number of workers started by the Decision Engine")
class StopState(enum.Enum):
NotFound = 1
Clean = 2
Terminated = 3
def _channel_preamble(name):
header = f"Channel: {name}"
rule = "=" * len(header)
return "\n" + rule + "\n" + header + "\n" + rule + "\n\n"
def _verify_redis_url(broker_url):
def _gauge(*args, **kwargs):
return Gauge(*args, **kwargs)
def test_gauge_set_value(Gauge):
"""Test setting a gauge value"""
g = Gauge("test_gauge_set_value", "test_gauge_set_value")
g.set(42)
assert g._value.get() == 42
import structlog
from kombu import Connection, Queue
from decisionengine.framework.dataspace import datablock
from decisionengine.framework.modules.logging_configDict import CHANNELLOGGERNAME
from decisionengine.framework.taskmanager.LatestMessages import LatestMessages
from decisionengine.framework.taskmanager.ProcessingState import ProcessingState, State
from decisionengine.framework.taskmanager.SourceProductCache import SourceProductCache
from decisionengine.framework.util.metrics import Gauge, Histogram
# Metrics for monitoring TaskManager
CHANNEL_STATE_GAUGE = Gauge(
"de_channel_state",
"Channel state",
[
"channel_name",
],
)
LOGICENGINE_RUN_GAUGE = Gauge(
"de_logicengine_last_run_timestamp_seconds",
"Last time a logicengine successfully ran",
[
"channel_name",
"logicengine_name",
],
)
TRANSFORM_RUN_GAUGE = Gauge(
"de_transform_last_run_timestamp_seconds",
from kombu import Connection, Queue
from kombu.pools import producers
from decisionengine.framework.modules import Module
from decisionengine.framework.modules.logging_configDict import LOGGERNAME
from decisionengine.framework.modules.Source import Source
from decisionengine.framework.taskmanager.module_graph import _create_module_instance
from decisionengine.framework.taskmanager.ProcessingState import State
from decisionengine.framework.util.metrics import Gauge
_DEFAULT_SCHEDULE = 300 # 5 minutes
SOURCE_ACQUIRE_GAUGE = Gauge(
"de_source_last_acquire_timestamp_seconds",
"Last time a source successfully ran its acquire function",
[
"source_name",
],
)
class SourceWorker(multiprocessing.Process):
"""
Provides interface to loadable modules an events to sycronise
execution
"""
def __init__(self, key, config, channel_name, exchange, broker_url):
"""
:type config: :obj:`dict`
:arg config: configuration dictionary describing the worker
"""