def test_put(self): test_input = (1, 2, 3) with LocalFileStorage(os.path.join(TEST_FOLDER, "bar")) as stor: stor.put(test_input) self.assertEqual(stor.get().get(), test_input) with LocalFileStorage(os.path.join(TEST_FOLDER, "bar")) as stor: self.assertEqual(stor.get().get(), test_input) with mock.patch("os.rename", side_effect=throw(Exception)): self.assertIsNone(stor.put(test_input))
def test_check_storage_size_links(self): test_input = (1, 2, 3) with LocalFileStorage(os.path.join(TEST_FOLDER, "asd4"), 1000) as stor: stor.put(test_input) with mock.patch("os.path.islink") as os_mock: os_mock.return_value = True self.assertTrue(stor._check_storage_size())
def __init__(self, **kwargs: Any) -> None: """Azure Monitor base exporter for OpenTelemetry. :param options: Exporter configuration options. :type options: ~azure.opentelemetry.exporter.azuremonitor.options.ExporterOptions :keyword str connection_string: The connection string to be used for authentication :rtype: None """ options = ExporterOptions(**kwargs) parsed_connection_string = ConnectionStringParser( options.connection_string) self._instrumentation_key = parsed_connection_string.instrumentation_key self._timeout = 10.0 # networking timeout in seconds temp_suffix = self._instrumentation_key or "" default_storage_path = os.path.join(tempfile.gettempdir(), TEMPDIR_PREFIX + temp_suffix) config = AzureMonitorClientConfiguration( parsed_connection_string.endpoint, **kwargs) policies = [ RequestIdPolicy(**kwargs), config.headers_policy, config.user_agent_policy, config.proxy_policy, ContentDecodePolicy(**kwargs), config.redirect_policy, config.retry_policy, config.authentication_policy, config.custom_hook_policy, config.logging_policy, # Explicitly disabling to avoid infinite loop of Span creation when data is exported # DistributedTracingPolicy(**kwargs), config.http_logging_policy or HttpLoggingPolicy(**kwargs) ] self.client = AzureMonitorClient( host=parsed_connection_string.endpoint, connection_timeout=self._timeout, policies=policies, **kwargs) self.storage = LocalFileStorage( path=default_storage_path, max_size=50 * 1024 * 1024, # Maximum size in bytes. maintenance_period=60, # Maintenance interval in seconds. retention_period=7 * 24 * 60 * 60, # Retention period in seconds )
def test_check_storage_size_error(self): test_input = (1, 2, 3) with LocalFileStorage(os.path.join(TEST_FOLDER, "asd5"), 1) as stor: with mock.patch("os.path.getsize", side_effect=throw(OSError)): stor.put(test_input) with mock.patch("os.path.islink") as os_mock: os_mock.return_value = True self.assertTrue(stor._check_storage_size())
def test_put_max_size(self): test_input = (1, 2, 3) with LocalFileStorage(os.path.join(TEST_FOLDER, "asd")) as stor: size_mock = mock.Mock() size_mock.return_value = False stor._check_storage_size = size_mock stor.put(test_input) self.assertEqual(stor.get(), None)
def test_get(self): now = _now() with LocalFileStorage(os.path.join(TEST_FOLDER, "foo")) as stor: stor.put((1, 2, 3), lease_period=10) with mock.patch( "azure.opentelemetry.exporter.azuremonitor._storage._now" ) as m: m.return_value = now - _seconds(30 * 24 * 60 * 60) stor.put((1, 2, 3)) stor.put((1, 2, 3), lease_period=10) with mock.patch("os.rename"): stor.put((1, 2, 3)) with mock.patch("os.rename"): stor.put((1, 2, 3)) with mock.patch("os.remove", side_effect=throw(Exception)): with mock.patch("os.rename", side_effect=throw(Exception)): self.assertIsNone(stor.get()) self.assertIsNone(stor.get())
class BaseExporter: """Azure Monitor base exporter for OpenTelemetry. :param options: Exporter configuration options. :type options: ~azure.opentelemetry.exporter.azuremonitor.options.ExporterOptions """ def __init__(self, **kwargs: Any) -> None: """Azure Monitor base exporter for OpenTelemetry. :param options: Exporter configuration options. :type options: ~azure.opentelemetry.exporter.azuremonitor.options.ExporterOptions :keyword str connection_string: The connection string to be used for authentication :rtype: None """ options = ExporterOptions(**kwargs) parsed_connection_string = ConnectionStringParser( options.connection_string) self._instrumentation_key = parsed_connection_string.instrumentation_key self._timeout = 10.0 # networking timeout in seconds temp_suffix = self._instrumentation_key or "" default_storage_path = os.path.join(tempfile.gettempdir(), TEMPDIR_PREFIX + temp_suffix) config = AzureMonitorClientConfiguration( parsed_connection_string.endpoint, **kwargs) policies = [ RequestIdPolicy(**kwargs), config.headers_policy, config.user_agent_policy, config.proxy_policy, ContentDecodePolicy(**kwargs), config.redirect_policy, config.retry_policy, config.authentication_policy, config.custom_hook_policy, config.logging_policy, # Explicitly disabling to avoid infinite loop of Span creation when data is exported # DistributedTracingPolicy(**kwargs), config.http_logging_policy or HttpLoggingPolicy(**kwargs) ] self.client = AzureMonitorClient( host=parsed_connection_string.endpoint, connection_timeout=self._timeout, policies=policies, **kwargs) self.storage = LocalFileStorage( path=default_storage_path, max_size=50 * 1024 * 1024, # Maximum size in bytes. maintenance_period=60, # Maintenance interval in seconds. retention_period=7 * 24 * 60 * 60, # Retention period in seconds ) def _transmit_from_storage(self) -> None: for blob in self.storage.gets(): # give a few more seconds for blob lease operation # to reduce the chance of race (for perf consideration) if blob.lease(self._timeout + 5): envelopes = [TelemetryItem(**x) for x in blob.get()] result = self._transmit(list(envelopes)) if result == ExportResult.FAILED_RETRYABLE: blob.lease(1) else: blob.delete() # pylint: disable=too-many-branches # pylint: disable=too-many-nested-blocks # pylint: disable=too-many-return-statements def _transmit(self, envelopes: List[TelemetryItem]) -> ExportResult: """ Transmit the data envelopes to the ingestion service. Returns an ExportResult, this function should never throw an exception. """ if len(envelopes) > 0: try: track_response = self.client.track(envelopes) if not track_response.errors: logger.info( "Transmission succeeded: Item received: %s. Items accepted: %s", track_response.items_received, track_response.items_accepted) return ExportResult.SUCCESS resend_envelopes = [] for error in track_response.errors: if _is_retryable_code(error.status_code): resend_envelopes.append(envelopes[error.index]) else: logger.error( "Data drop %s: %s %s.", error.status_code, error.message, envelopes[error.index] if error.index is not None else "", ) if resend_envelopes: envelopes_to_store = [ x.as_dict() for x in resend_envelopes ] self.storage.put(envelopes_to_store) return ExportResult.FAILED_RETRYABLE except HttpResponseError as response_error: if _is_retryable_code(response_error.status_code): return ExportResult.FAILED_RETRYABLE return ExportResult.FAILED_NOT_RETRYABLE except ServiceRequestError as request_error: # Errors when we're fairly sure that the server did not receive the # request, so it should be safe to retry. logger.warning("Retrying due to server request error: %s.", request_error) return ExportResult.FAILED_RETRYABLE except Exception as ex: logger.error( "Envelopes could not be exported and are not retryable: %s.", ex) return ExportResult.FAILED_NOT_RETRYABLE return ExportResult.FAILED_NOT_RETRYABLE # No spans to export return ExportResult.SUCCESS
def test_get_nothing(self): with LocalFileStorage(os.path.join(TEST_FOLDER, "test", "a")) as stor: pass with LocalFileStorage(os.path.join(TEST_FOLDER, "test")) as stor: self.assertIsNone(stor.get())
def test_maintanence_routine(self): with mock.patch("os.makedirs") as m: with LocalFileStorage(os.path.join(TEST_FOLDER, "baz")) as stor: m.return_value = None with mock.patch("os.makedirs", side_effect=throw(Exception)): stor = LocalFileStorage(os.path.join(TEST_FOLDER, "baz")) stor.close() with mock.patch("os.listdir", side_effect=throw(Exception)): stor = LocalFileStorage(os.path.join(TEST_FOLDER, "baz")) stor.close() with LocalFileStorage(os.path.join(TEST_FOLDER, "baz")) as stor: with mock.patch("os.listdir", side_effect=throw(Exception)): stor._maintenance_routine() with mock.patch("os.path.isdir", side_effect=throw(Exception)): stor._maintenance_routine()
def test_check_storage_size_no_files(self): with LocalFileStorage(os.path.join(TEST_FOLDER, "asd3"), 1000) as stor: self.assertTrue(stor._check_storage_size())
def test_check_storage_size_not_full(self): test_input = (1, 2, 3) with LocalFileStorage(os.path.join(TEST_FOLDER, "asd3"), 1000) as stor: stor.put(test_input) self.assertTrue(stor._check_storage_size())