def stream(self, metrics):
LOG.debug("Metrics collection")
metrics_to_stream = []
metric = snap.Metric(namespace=[
snap.NamespaceElement(value="intel"),
snap.NamespaceElement(value="streaming"),
snap.NamespaceElement(value="random"),
snap.NamespaceElement(value="int")
],
version=1,
tags={"mtype": "counter"},
description="Random int",
data=random.randint(1, 100),
timestamp=time.time())
metrics_to_stream.append(metric)
metric = snap.Metric(namespace=[
snap.NamespaceElement(value="intel"),
snap.NamespaceElement(value="streaming"),
snap.NamespaceElement(value="random"),
snap.NamespaceElement(value="float")
],
version=1,
tags={"mtype": "counter"},
description="Random float",
data=random.random(),
timestamp=time.time())
metrics_to_stream.append(metric)
time.sleep(1)
return metrics_to_stream
def update_catalog(self, config):
LOG.debug("GetMetricTypes called")
metrics = []
for key in ("float64", "int64", "string"):
metric = snap.Metric(
namespace=[
snap.NamespaceElement(value="random"),
snap.NamespaceElement(value=key)
],
version=1,
tags={"mtype": "gauge"},
description="Random {}".format(key),
)
metrics.append(metric)
metric = snap.Metric(version=1, Description="dynamic element example")
# adds namespace elements (static and dynamic) via namespace methods
metric.namespace.add_static_element("random")
metric.namespace.add_dynamic_element("pid", "current pid")
metric.namespace.add_static_element("uid")
metrics.append(metric)
# metric is added with the namespace defined in the constructor
metric = snap.Metric(namespace=[
snap.NamespaceElement(value="random"),
snap.NamespaceElement(name="pid", description="current pid"),
snap.NamespaceElement(value="gid")
],
description="dynamic element example",
version=1)
metrics.append(metric)
return metrics
def test_process(processor_client):
now = time.time()
metrics = [
snap.Metric(
namespace=[
snap.NamespaceElement(value="org"),
snap.NamespaceElement(value="metric"),
snap.NamespaceElement(value="foo")
],
version=1,
unit="some unit",
description="some description",
timestamp=now,
)
]
config = snap.ConfigMap(foo="bar")
reply = processor_client.Process(
_ProcessArg(metrics=metrics, config=config).pb)
assert reply.error == ""
assert len(reply.metrics) == 1
for m in reply.metrics:
assert "processed" in m.Tags
assert m.Tags["processed"] == "true"
assert "foo" in m.Tags
assert m.Tags["foo"] == "bar"
def test_stream_max_metrics_buffer():
sys.stdout = ThreadPrinter()
sys.argv = ["", '{"LogLevel": 1, "PingTimeoutDuration": 5000}']
col = MockStreamCollector("MyStreamCollector", 99)
col.start()
t_end = time.time() + 5
# wait for our collector to print its preamble
while len(sys.stdout.lines) == 0 and time.time() < t_end:
time.sleep(.1)
resp = json.loads(sys.stdout.lines[0])
client = StreamCollectorStub(
grpc.insecure_channel(resp["ListenAddress"]))
metric = snap.Metric(
namespace=[snap.NamespaceElement(value="intel"),
snap.NamespaceElement(value="streaming"),
snap.NamespaceElement(value="random"),
snap.NamespaceElement(value="int")],
version=1,
config={"max-metrics-buffer": 5},
unit="some unit",
description="some description")
col_arg = CollectArg(metric).pb
mtr = iter([col_arg])
metrics = client.StreamMetrics(mtr)
start_waiting_for_new_metric = time.time()
a = next(metrics)
retrieve_metric_time = time.time()
assert round(retrieve_metric_time - start_waiting_for_new_metric) == 5
assert len(a.Metrics_Reply.metrics) == 5
col.stop()
def test_multiple_stream():
sys.stdout = ThreadPrinter()
sys.argv = ["", '{"LogLevel": 1, "PingTimeoutDuration": 5000}']
col = MockStreamCollector("MyStreamCollector", 99)
col.start()
t_end = time.time() + 5
# wait for our collector to print its preamble
while len(sys.stdout.lines) == 0 and time.time() < t_end:
time.sleep(.1)
resp = json.loads(sys.stdout.lines[0])
client = StreamCollectorStub(
grpc.insecure_channel(resp["ListenAddress"]))
metric = snap.Metric(
namespace=[snap.NamespaceElement(value="intel"),
snap.NamespaceElement(value="streaming"),
snap.NamespaceElement(value="random"),
snap.NamespaceElement(value="int")],
version=1,
unit="some unit",
description="some description",
config={"send_multiple": True}
)
mtr = iter([CollectArg(metric).pb])
metrics = client.StreamMetrics(mtr)
a = next(metrics)
assert len(a.Metrics_Reply.metrics) == 3
col.stop()
def update_catalog(self, config):
LOG.debug("GetMetricTypes called")
while self.init_lock:
if not self.oslo_listener:
LOG.debug("Starting oslo.messaging listener")
self.oslo_listener = messaging.get_notification_listener(
messaging.get_transport(
cfg.CONF, config["transport_url"]),
[messaging.Target(topic='notifications')],
[NotificationHandler()], 'threading', allow_requeue=True)
self.oslo_listener.start()
metrics = []
for key in ("cpu_cstate", "io_bandwith", "memory_bandwith"):
metric = snap.Metric(
namespace=[
snap.NamespaceElement(value="intel"),
snap.NamespaceElement(value="node_manager"),
snap.NamespaceElement(value="cups"),
snap.NamespaceElement(value=key)
],
version=1,
tags={"mtype": "gauge"},
description="CUPS {}".format(key.replace('_', ' ')),
)
metric.namespace.add_dynamic_element("uuid", "node UUID")
metrics.append(metric)
return metrics
def test_ovs_collect(self, test_topology):
plugin = Ovs("ovs", 1)
metrics = plugin.collect([
snap.Metric(namespace=[
snap.NamespaceElement(value="intel"),
snap.NamespaceElement(value="ovs"),
snap.NamespaceElement(value="*"),
snap.NamespaceElement(value="tx_bytes")
]),
snap.Metric(namespace=[
snap.NamespaceElement(value="intel"),
snap.NamespaceElement(value="ovs"),
snap.NamespaceElement(value="*"),
snap.NamespaceElement(value="rx_bytes")
]),
])
assert len(metrics) > 0
def process(self, metric):
snap_metric = snap.Metric(
namespace=("diamond", metric.getCollectorPath(), metric.getMetricPath()),
tags={
"plugin_running_on": metric.host,
"metric_type": metric.metric_type,
},
timestamp=time.time(),
data=metric.value
)
self._queue.put(snap_metric)
def update_catalog(self, config):
metrics = []
metric = snap.Metric(version=1)
metric.namespace.add_static_element("mfms") # /0
metric.namespace.add_static_element("mdstat") # /1
metric.namespace.add_dynamic_element("raid", "raid device name") # /2
metric.namespace.add_dynamic_element("disk", "raid disk name") # /3
metric.namespace.add_static_element("health") # /4
metrics.append(metric)
return metrics
def update_catalog(self, config):
metrics = [
snap.Metric(
namespace=[
snap.NamespaceElement(value="intel"),
snap.NamespaceElement(value="streaming"),
snap.NamespaceElement(value="random"),
snap.NamespaceElement(value="int")
],
unit="some unit",
description="some description",
)
]
return metrics
def collect(self, metrics):
metrics_return = []
ts_now = time.time()
md_devs = mdstat.parse()['devices']
for (md_dev_name, md_dev_info) in md_devs.items():
metric = snap.Metric(namespace=[i for i in metrics[0].namespace])
metric.namespace[2].value = md_dev_name
metric.namespace[3].value = "DISKS"
metric.data = md_dev_info["status"]["raid_disks"] - len(
md_dev_info["disks"])
metric.timestamp = ts_now
metric.tags['host'] = self.hostname
metrics_return.append(metric)
for (disk_name, disk_info) in md_dev_info['disks'].items():
metric = snap.Metric(
namespace=[i for i in metrics[0].namespace])
metric.namespace[2].value = md_dev_name
metric.namespace[3].value = disk_name
metric.data = 1 if disk_info['faulty'] else 0
metric.timestamp = ts_now
metric.tags['host'] = self.hostname
metrics_return.append(metric)
return metrics_return
def update_catalog(self, config):
LOG.debug("GetMetricTypes called")
metrics = []
for key in ("float64", "int64", "string"):
metric = snap.Metric(
namespace=[
snap.NamespaceElement(value="random"),
snap.NamespaceElement(value=key)
],
version=1,
tags={"mtype": "gauge"},
Description="Random {}".format(key),
)
metrics.append(metric)
return metrics
def update_catalog(self, config):
now = time.time()
metrics = [
snap.Metric(
namespace=[
snap.NamespaceElement(value="acme"),
snap.NamespaceElement(value="sk8"),
snap.NamespaceElement(value="matix")
],
unit="some unit",
description="some description",
timestamp=now,
)
]
return metrics
def collect(self, metrics):
metricsToReturn = []
# set the time before the loop in case the time changes as the metric
# values are being set
ts_now = time.time()
# loop through each device and each attribute on the device and store
# the value to metric
for dev in self.devices:
# dev.attributes is the list of S.M.A.R.T. attributes avaible on
# each device, may change depending on the devide
for att in dev.attributes:
if att is not None:
for metric in metrics:
# sets the metricTeReturn to the metrics class which
# inherits the namespace, unit, and tags from the
# metric in metrics
_metrics = snap.Metric(
namespace=[i for i in metric.namespace],
unit=metric.unit)
_metrics.tags = [(k, v)
for k, v in metric.tags.items()]
# set the dynamic device name
_metrics.namespace[3].value = dev.name
# set the dynamic attribute name
_metrics.namespace[4].value = att.name
# store the value into the metric data
if _metrics.namespace[5].value == "threshold":
_metrics.data = att.thresh
if _metrics.namespace[5].value == "value":
_metrics.data = att.value
if _metrics.namespace[5].value == "whenfailed":
_metrics.data = att.when_failed
if _metrics.namespace[5].value == "worst":
_metrics.data = att.worst
if _metrics.namespace[5].value == "type":
_metrics.data = att.type
if _metrics.namespace[5].value == "updated":
_metrics.data = att.updated
if _metrics.namespace[5].value == "raw":
_metrics.data = att.raw
if _metrics.namespace[5].value == "num":
_metrics.data = att.num
# store the time stamp for each metric
_metrics.timestamp = ts_now
metricsToReturn.append(_metrics)
return metricsToReturn
def stream(self, requested_metrics):
delay = 1
if "stream_delay" in requested_metrics[0].config:
delay = requested_metrics[0].config["stream_delay"]
now = time.time()
metric = snap.Metric(namespace=[
snap.NamespaceElement(value="intel"),
snap.NamespaceElement(value="streaming"),
snap.NamespaceElement(value="random"),
snap.NamespaceElement(value="int")
],
version=1,
tags={"mtype": "counter"},
description="some description",
timestamp=now,
data=200)
time.sleep(delay)
return metric
def update_catalog(self, config):
"""
Updates snap catalog
@param config: snap config instance
@return: list of metrics
"""
LOG.debug("GetMetricTypes called")
metrics = []
for i in metrics_table:
metric = snap.Metric(version=self.meta.version,
Description="OVS list of dynamic devices")
metric.namespace.add_static_element("intel")
metric.namespace.add_static_element("ovs")
metric.namespace.add_dynamic_element("device", "device name")
metric.namespace.add_static_element(i)
metrics.append(metric)
return metrics
def test_publish(publisher_client):
now = time.time()
metrics = [
snap.Metric(
namespace=[
snap.NamespaceElement(value="org"),
snap.NamespaceElement(value="metric"),
snap.NamespaceElement(value="foo")
],
version=1,
unit="some unit",
description="some description",
timestamp=now,
)
]
config = snap.ConfigMap(foo="bar", port=911, debug=True, availability=99.9)
reply = publisher_client.Publish(
_PublishArg(metrics=metrics, config=config).pb)
assert reply.error == ""
def update_catalog(self, config):
LOG.debug("GetMetricTypes called")
metrics = []
# adds namespace elements (static and dynamic) via namespace methods
for i in ("threshold", "value", "whenfailed", "worst", "type",
"updated", "raw", "num"):
metric = snap.Metric(version=1,
Description="SMARTMON list of dynamic devices"
" and attributes")
metric.namespace.add_static_element("intel")
metric.namespace.add_static_element("smartmon")
metric.namespace.add_static_element("devices")
# dynamic elements which are captured from the smartmontool
metric.namespace.add_dynamic_element("device", "device name")
metric.namespace.add_dynamic_element("attribute", "attribute name")
# values of the attributes to collect
metric.namespace.add_static_element(i)
metrics.append(metric)
return metrics
def collect(self, metrics):
"""
Collects metrics
@param metrics: list of snap metric namespaces
@return: list of metrics
"""
metrics_to_return = []
ts_now = time.time()
topology = get_switch_topology()
interfaces = get_interfaces(topology)
for iface in interfaces:
for metric in metrics:
tmp_metric = snap.Metric(
namespace=[i for i in metric.namespace], unit=metric.unit)
tmp_metric.tags = [(k, v) for k, v in metric.tags.items()]
tmp_metric.namespace[2].value = iface[0]
data = iface[1].get(metric.namespace[3].value, None)
if not data:
continue
tmp_metric.timestamp = ts_now
metrics_to_return.append(metric)
return metrics_to_return
def update_catalog(self, config):
LOG.debug("GetMetricTypes called")
metrics = []
keys = ("float64", "int64", "string")
self._args.required_config = False
#print("self._args.required_config", self._args.required_config)
#print("snap.Publisher ")
for key in ("float", "int"):
metric = snap.Metric(
namespace=[
snap.NamespaceElement(value="intel"),
snap.NamespaceElement(value="logs"),
snap.NamespaceElement(value="*")
],
data="str",
version=1,
tags={"mtype": "gauge"},
description="http publisher {}".format(key),
)
metrics.append(metric)
return metrics
def test_collect(collector_client):
now = time.time()
metric = snap.Metric(
namespace=[snap.NamespaceElement(value="org"),
snap.NamespaceElement(value="metric"),
snap.NamespaceElement(value="foo")],
version=1,
config={"foo": "bar"},
unit="some unit",
description="some description",
timestamp=now)
reply = collector_client.CollectMetrics(MetricsArg(metric).pb)
assert reply.error == ''
assert len(reply.metrics) == 1
assert reply.metrics[0].Version == 2
assert reply.metrics[0].float64_data == 99.9
assert snap.Metric(pb=reply.metrics[0]).data == 99.9
reply2 = collector_client.CollectMetrics(MetricsArg(metric).pb)
assert (reply2.metrics[0].Timestamp.sec +
(reply2.metrics[0].Timestamp.nsec * 10 ** -9)) > (
reply.metrics[0].Timestamp.sec +
(reply.metrics[0].Timestamp.nsec * 10 ** -9))
# collect bytes
metric.config.clear()
metric.config["bytes"] = True
reply = collector_client.CollectMetrics(MetricsArg(metric).pb)
assert reply.error == ''
assert snap.Metric(pb=reply.metrics[0]).data == 'qwerty'
# collect string
metric.config.clear()
metric.config["string"] = True
reply = collector_client.CollectMetrics(MetricsArg(metric).pb)
assert reply.error == ''
assert snap.Metric(pb=reply.metrics[0]).data == "qwerty"
assert isinstance(snap.Metric(pb=reply.metrics[0]).data, basestring)
# collect int
metric.config.clear()
metric.config["int32"] = True
reply = collector_client.CollectMetrics(MetricsArg(metric).pb)
assert reply.error == ''
assert snap.Metric(pb=reply.metrics[0]).data == 99
assert isinstance(snap.Metric(pb=reply.metrics[0]).data, bigint)
# collect int64 (long)
metric.config.clear()
metric.config["int64"] = True
reply = collector_client.CollectMetrics(MetricsArg(metric).pb)
assert reply.error == ''
assert snap.Metric(pb=reply.metrics[0]).data == 99
assert isinstance(snap.Metric(pb=reply.metrics[0]).data, bigint)
# collect int
metric.config.clear()
metric.config["bool"] = True
reply = collector_client.CollectMetrics(MetricsArg(metric).pb)
assert reply.error == ''
assert bool(snap.Metric(pb=reply.metrics[0]).data) is True
