def test_sync(self, controller_mock):
"""
Test if the collector syncs our sample.
"""
# Mock out Collect.future
mock = unittest.mock.Mock()
mock.Collect.future.side_effect = [
unittest.mock.Mock(),
unittest.mock.Mock(),
unittest.mock.Mock()
]
controller_mock.side_effect = [mock, mock, mock]
# Call with no samples
service_name = "test"
self._collector.sync(service_name)
controller_mock.Collect.future.assert_not_called()
self._collector._loop.stop()
# Call with new samples to send
samples = [MetricFamily(name="1234")]
self._collector._samples_for_service[service_name].extend(samples)
with unittest.mock.patch('snowflake.snowflake') as mock_snowflake:
mock_snowflake.side_effect = lambda: self.gateway_id
self._collector.sync(service_name)
mock.Collect.future.assert_called_once_with(
MetricsContainer(gatewayId=self.gateway_id, family=samples),
self.timeout)
self.assertCountEqual(
self._collector._samples_for_service[service_name], [])
# Reduce max msg size to trigger msg chunking
self._collector.grpc_max_msg_size_bytes = 1500
samples = self._generate_samples(140)
self._collector._samples_for_service[service_name].extend(samples)
chunk1 = samples[:70]
chunk2 = samples[70:140]
with unittest.mock.patch('snowflake.snowflake') as mock_snowflake:
mock_snowflake.side_effect = lambda: self.gateway_id
self._collector.sync(service_name)
mock.Collect.future.assert_any_call(
MetricsContainer(gatewayId=self.gateway_id, family=chunk1),
self.timeout)
mock.Collect.future.assert_any_call(
MetricsContainer(gatewayId=self.gateway_id, family=chunk2),
self.timeout)
self.assertCountEqual(
self._collector._samples_for_service[service_name], [])
def test_sync(self, controller_mock):
"""
Test if the collector syncs our sample.
"""
# Mock out Collect.future
mock = unittest.mock.Mock()
mock.Collect.future.side_effect = [unittest.mock.Mock()]
controller_mock.side_effect = [mock]
# Call with no samples
self._collector.sync()
controller_mock.Collect.future.assert_not_called()
self._collector._loop.stop()
# Call with new samples to send and some to retry
samples = [MetricFamily(name="1234")]
self._collector._samples.extend(samples)
self._collector._retry_queue.extend(samples)
with unittest.mock.patch('snowflake.snowflake') as mock_snowflake:
mock_snowflake.side_effect = lambda: self.gateway_id
self._collector.sync()
mock.Collect.future.assert_called_once_with(
MetricsContainer(gatewayId=self.gateway_id, family=samples * 2),
self.timeout)
self.assertCountEqual(self._collector._samples, [])
self.assertCountEqual(self._collector._retry_queue, [])
def sync(self):
"""
Synchronizes sample queue to cloud and reschedules sync loop
"""
if self._samples:
chan = ServiceRegistry.get_rpc_channel('metricsd',
ServiceRegistry.CLOUD)
client = MetricsControllerStub(chan)
if self.post_processing_fn:
# If services wants to, let it run a postprocessing function
# If we throw an exception here, we'll have no idea whether
# something was postprocessed or not, so I guess try and make it
# idempotent? #m sevchicken
self.post_processing_fn(self._samples)
samples = self._retry_queue + self._samples
metrics_container = MetricsContainer(
gatewayId=snowflake.snowflake(), family=samples)
future = client.Collect.future(metrics_container,
self.grpc_timeout)
future.add_done_callback(
lambda future: self._loop.call_soon_threadsafe(
self.sync_done, samples, future))
self._retry_queue.clear()
self._samples.clear()
self._loop.call_later(self.sync_interval, self.sync)
def _package_and_send_metrics(
self, metrics: [metrics_pb2.MetricFamily],
target: ScrapeTarget,
) -> None:
"""
Send parsed and protobuf-converted metrics to cloud.
"""
chan = ServiceRegistry.get_rpc_channel(
'metricsd',
ServiceRegistry.CLOUD,
grpc_options=self._grpc_options,
)
client = MetricsControllerStub(chan)
for chunk in self._chunk_samples(metrics):
metrics_container = MetricsContainer(
gatewayId=snowflake.snowflake(),
family=chunk,
)
future = client.Collect.future(
metrics_container,
self.grpc_timeout,
)
future.add_done_callback(
lambda future:
self._loop.call_soon_threadsafe(
self.scrape_done, future, target,
),
)
self._loop.call_later(
target.interval,
self.scrape_prometheus_target, target,
)
def sync(self, service_name):
"""
Synchronizes sample queue for specific service to cloud and reschedules
sync loop
"""
if service_name in self._samples_for_service and \
self._samples_for_service[service_name]:
chan = ServiceRegistry.get_rpc_channel(
'metricsd',
ServiceRegistry.CLOUD,
grpc_options=self._grpc_options)
client = MetricsControllerStub(chan)
if self.post_processing_fn:
# If services wants to, let it run a postprocessing function
# If we throw an exception here, we'll have no idea whether
# something was postprocessed or not, so I guess try and make it
# idempotent? #m sevchicken
self.post_processing_fn(
self._samples_for_service[service_name])
samples = self._samples_for_service[service_name]
sample_chunks = self._chunk_samples(samples)
for idx, chunk in enumerate(sample_chunks):
metrics_container = MetricsContainer(
gatewayId=snowflake.snowflake(), family=chunk)
future = client.Collect.future(metrics_container,
self.grpc_timeout)
future.add_done_callback(
self._make_sync_done_func(service_name, idx))
self._samples_for_service[service_name].clear()
self._loop.call_later(self.sync_interval, self.sync, service_name)
def GetMetrics(self, request, context):
"""
Collects timeseries samples from prometheus python client on this
process
"""
metrics = MetricsContainer()
metrics.family.extend(get_metrics())
return metrics
def test_collect_start_time(self):
"""
Test if the collector syncs our sample.
"""
mock = unittest.mock.MagicMock()
start_metric = Metric()
start_metric.gauge.value = calendar.timegm(time.gmtime()) - 1
start_time = MetricFamily(
name=str(metricsd_pb2.process_start_time_seconds),
metric=[start_metric],
)
samples = [start_time]
service_name = "test"
self._collector._samples_for_service[service_name].clear()
mock.result.side_effect = [MetricsContainer(family=samples)]
mock.exception.side_effect = [False]
self._collector.collect_done('test', mock)
# should have uptime, start time, and collection success
self.assertEqual(
len(self._collector._samples_for_service[service_name]),
3,
)
uptime_list = [
fam for fam in self._collector._samples_for_service[service_name]
if fam.name == str(metricsd_pb2.process_uptime_seconds)
]
self.assertEqual(len(uptime_list), 1)
self.assertEqual(len(uptime_list[0].metric), 1)
self.assertGreater(uptime_list[0].metric[0].gauge.value, 0)
# ensure no exceptions with empty metric
empty = MetricFamily(name=str(metricsd_pb2.process_start_time_seconds))
samples = [empty]
self._collector._samples_for_service[service_name].clear()
mock.result.side_effect = [MetricsContainer(family=samples)]
mock.exception.side_effect = [False]
try:
self._collector.collect_done('test', mock)
except Exception: # pylint: disable=broad-except
self.fail("Collection with empty metric should not have failed")
def test_collect(self):
"""
Test if the collector syncs our sample.
"""
mock = unittest.mock.MagicMock()
samples = [MetricFamily(name="2345")]
self._collector._samples.clear()
self._collector._samples.extend(samples)
mock.result.side_effect = [MetricsContainer(family=samples)]
mock.exception.side_effect = [False]
self._collector.collect_done('test', mock)
# Should dequeue sample from the left, and enqueue on right
# collector should add one more metric for collection success/failure
self.assertEqual(len(self._collector._samples), len(samples * 2) + 1)
def sync(self):
"""
Synchronizes sample queue to cloud and reschedules sync loop
"""
if self._samples:
chan = ServiceRegistry.get_rpc_channel('metricsd',
ServiceRegistry.CLOUD)
client = MetricsControllerStub(chan)
samples = self._retry_queue + self._samples
metrics_container = MetricsContainer(
gatewayId=snowflake.snowflake(), family=samples)
future = client.Collect.future(metrics_container,
self.grpc_timeout)
future.add_done_callback(
lambda future: self._loop.call_soon_threadsafe(
self.sync_done, samples, future))
self._retry_queue.clear()
self._samples.clear()
self._loop.call_later(self.sync_interval, self.sync)
