def test_sync(self, controller_mock): """ Test if the collector syncs our sample. """ # Mock out Collect.future mock = unittest.mock.Mock() mock.Collect.future.side_effect = [ unittest.mock.Mock(), unittest.mock.Mock(), unittest.mock.Mock() ] controller_mock.side_effect = [mock, mock, mock] # Call with no samples service_name = "test" self._collector.sync(service_name) controller_mock.Collect.future.assert_not_called() self._collector._loop.stop() # Call with new samples to send samples = [MetricFamily(name="1234")] self._collector._samples_for_service[service_name].extend(samples) with unittest.mock.patch('snowflake.snowflake') as mock_snowflake: mock_snowflake.side_effect = lambda: self.gateway_id self._collector.sync(service_name) mock.Collect.future.assert_called_once_with( MetricsContainer(gatewayId=self.gateway_id, family=samples), self.timeout) self.assertCountEqual( self._collector._samples_for_service[service_name], []) # Reduce max msg size to trigger msg chunking self._collector.grpc_max_msg_size_bytes = 1500 samples = self._generate_samples(140) self._collector._samples_for_service[service_name].extend(samples) chunk1 = samples[:70] chunk2 = samples[70:140] with unittest.mock.patch('snowflake.snowflake') as mock_snowflake: mock_snowflake.side_effect = lambda: self.gateway_id self._collector.sync(service_name) mock.Collect.future.assert_any_call( MetricsContainer(gatewayId=self.gateway_id, family=chunk1), self.timeout) mock.Collect.future.assert_any_call( MetricsContainer(gatewayId=self.gateway_id, family=chunk2), self.timeout) self.assertCountEqual( self._collector._samples_for_service[service_name], [])
def test_sync(self, controller_mock): """ Test if the collector syncs our sample. """ # Mock out Collect.future mock = unittest.mock.Mock() mock.Collect.future.side_effect = [unittest.mock.Mock()] controller_mock.side_effect = [mock] # Call with no samples self._collector.sync() controller_mock.Collect.future.assert_not_called() self._collector._loop.stop() # Call with new samples to send and some to retry samples = [MetricFamily(name="1234")] self._collector._samples.extend(samples) self._collector._retry_queue.extend(samples) with unittest.mock.patch('snowflake.snowflake') as mock_snowflake: mock_snowflake.side_effect = lambda: self.gateway_id self._collector.sync() mock.Collect.future.assert_called_once_with( MetricsContainer(gatewayId=self.gateway_id, family=samples * 2), self.timeout) self.assertCountEqual(self._collector._samples, []) self.assertCountEqual(self._collector._retry_queue, [])
def sync(self): """ Synchronizes sample queue to cloud and reschedules sync loop """ if self._samples: chan = ServiceRegistry.get_rpc_channel('metricsd', ServiceRegistry.CLOUD) client = MetricsControllerStub(chan) if self.post_processing_fn: # If services wants to, let it run a postprocessing function # If we throw an exception here, we'll have no idea whether # something was postprocessed or not, so I guess try and make it # idempotent? #m sevchicken self.post_processing_fn(self._samples) samples = self._retry_queue + self._samples metrics_container = MetricsContainer( gatewayId=snowflake.snowflake(), family=samples) future = client.Collect.future(metrics_container, self.grpc_timeout) future.add_done_callback( lambda future: self._loop.call_soon_threadsafe( self.sync_done, samples, future)) self._retry_queue.clear() self._samples.clear() self._loop.call_later(self.sync_interval, self.sync)
def _package_and_send_metrics( self, metrics: [metrics_pb2.MetricFamily], target: ScrapeTarget, ) -> None: """ Send parsed and protobuf-converted metrics to cloud. """ chan = ServiceRegistry.get_rpc_channel( 'metricsd', ServiceRegistry.CLOUD, grpc_options=self._grpc_options, ) client = MetricsControllerStub(chan) for chunk in self._chunk_samples(metrics): metrics_container = MetricsContainer( gatewayId=snowflake.snowflake(), family=chunk, ) future = client.Collect.future( metrics_container, self.grpc_timeout, ) future.add_done_callback( lambda future: self._loop.call_soon_threadsafe( self.scrape_done, future, target, ), ) self._loop.call_later( target.interval, self.scrape_prometheus_target, target, )
def sync(self, service_name): """ Synchronizes sample queue for specific service to cloud and reschedules sync loop """ if service_name in self._samples_for_service and \ self._samples_for_service[service_name]: chan = ServiceRegistry.get_rpc_channel( 'metricsd', ServiceRegistry.CLOUD, grpc_options=self._grpc_options) client = MetricsControllerStub(chan) if self.post_processing_fn: # If services wants to, let it run a postprocessing function # If we throw an exception here, we'll have no idea whether # something was postprocessed or not, so I guess try and make it # idempotent? #m sevchicken self.post_processing_fn( self._samples_for_service[service_name]) samples = self._samples_for_service[service_name] sample_chunks = self._chunk_samples(samples) for idx, chunk in enumerate(sample_chunks): metrics_container = MetricsContainer( gatewayId=snowflake.snowflake(), family=chunk) future = client.Collect.future(metrics_container, self.grpc_timeout) future.add_done_callback( self._make_sync_done_func(service_name, idx)) self._samples_for_service[service_name].clear() self._loop.call_later(self.sync_interval, self.sync, service_name)
def GetMetrics(self, request, context): """ Collects timeseries samples from prometheus python client on this process """ metrics = MetricsContainer() metrics.family.extend(get_metrics()) return metrics
def test_collect_start_time(self): """ Test if the collector syncs our sample. """ mock = unittest.mock.MagicMock() start_metric = Metric() start_metric.gauge.value = calendar.timegm(time.gmtime()) - 1 start_time = MetricFamily( name=str(metricsd_pb2.process_start_time_seconds), metric=[start_metric], ) samples = [start_time] service_name = "test" self._collector._samples_for_service[service_name].clear() mock.result.side_effect = [MetricsContainer(family=samples)] mock.exception.side_effect = [False] self._collector.collect_done('test', mock) # should have uptime, start time, and collection success self.assertEqual( len(self._collector._samples_for_service[service_name]), 3, ) uptime_list = [ fam for fam in self._collector._samples_for_service[service_name] if fam.name == str(metricsd_pb2.process_uptime_seconds) ] self.assertEqual(len(uptime_list), 1) self.assertEqual(len(uptime_list[0].metric), 1) self.assertGreater(uptime_list[0].metric[0].gauge.value, 0) # ensure no exceptions with empty metric empty = MetricFamily(name=str(metricsd_pb2.process_start_time_seconds)) samples = [empty] self._collector._samples_for_service[service_name].clear() mock.result.side_effect = [MetricsContainer(family=samples)] mock.exception.side_effect = [False] try: self._collector.collect_done('test', mock) except Exception: # pylint: disable=broad-except self.fail("Collection with empty metric should not have failed")
def test_collect(self): """ Test if the collector syncs our sample. """ mock = unittest.mock.MagicMock() samples = [MetricFamily(name="2345")] self._collector._samples.clear() self._collector._samples.extend(samples) mock.result.side_effect = [MetricsContainer(family=samples)] mock.exception.side_effect = [False] self._collector.collect_done('test', mock) # Should dequeue sample from the left, and enqueue on right # collector should add one more metric for collection success/failure self.assertEqual(len(self._collector._samples), len(samples * 2) + 1)
def sync(self): """ Synchronizes sample queue to cloud and reschedules sync loop """ if self._samples: chan = ServiceRegistry.get_rpc_channel('metricsd', ServiceRegistry.CLOUD) client = MetricsControllerStub(chan) samples = self._retry_queue + self._samples metrics_container = MetricsContainer( gatewayId=snowflake.snowflake(), family=samples) future = client.Collect.future(metrics_container, self.grpc_timeout) future.add_done_callback( lambda future: self._loop.call_soon_threadsafe( self.sync_done, samples, future)) self._retry_queue.clear() self._samples.clear() self._loop.call_later(self.sync_interval, self.sync)