def test_retry_policy_does_not_mark_null_locations_unavailable(self): self.original_get_database_account = cosmos_client_connection.CosmosClientConnection.GetDatabaseAccount cosmos_client_connection.CosmosClientConnection.GetDatabaseAccount = self.mock_get_database_account client = cosmos_client_connection.CosmosClientConnection( self.DEFAULT_ENDPOINT, {'masterKey': self.MASTER_KEY}, None, documents.ConsistencyLevel.Eventual) endpoint_manager = global_endpoint_manager._GlobalEndpointManager( client) self.original_mark_endpoint_unavailable_for_read_function = endpoint_manager.mark_endpoint_unavailable_for_read endpoint_manager.mark_endpoint_unavailable_for_read = self._mock_mark_endpoint_unavailable_for_read self.original_mark_endpoint_unavailable_for_write_function = endpoint_manager.mark_endpoint_unavailable_for_write endpoint_manager.mark_endpoint_unavailable_for_write = self._mock_mark_endpoint_unavailable_for_write self.original_resolve_service_endpoint = endpoint_manager.resolve_service_endpoint endpoint_manager.resolve_service_endpoint = self._mock_resolve_service_endpoint # Read and write counters count the number of times the endpoint manager's # mark_endpoint_unavailable_for_read() and mark_endpoint_unavailable_for_read() # functions were called. When a 'None' location is returned by resolve_service_endpoint(), # these functions should not be called self._read_counter = 0 self._write_counter = 0 request = _RequestObject(http_constants.ResourceType.Document, documents._OperationType.Read) endpointDiscovery_retry_policy = endpoint_discovery_retry_policy._EndpointDiscoveryRetryPolicy( documents.ConnectionPolicy(), endpoint_manager, request) endpointDiscovery_retry_policy.ShouldRetry( errors.HTTPFailure(http_constants.StatusCodes.FORBIDDEN)) self.assertEqual(self._read_counter, 0) self.assertEqual(self._write_counter, 0) self._read_counter = 0 self._write_counter = 0 request = _RequestObject(http_constants.ResourceType.Document, documents._OperationType.Create) endpointDiscovery_retry_policy = endpoint_discovery_retry_policy._EndpointDiscoveryRetryPolicy( documents.ConnectionPolicy(), endpoint_manager, request) endpointDiscovery_retry_policy.ShouldRetry( errors.HTTPFailure(http_constants.StatusCodes.FORBIDDEN)) self.assertEqual(self._read_counter, 0) self.assertEqual(self._write_counter, 0) endpoint_manager.mark_endpoint_unavailable_for_read = self.original_mark_endpoint_unavailable_for_read_function endpoint_manager.mark_endpoint_unavailable_for_write = self.original_mark_endpoint_unavailable_for_write_function cosmos_client_connection.CosmosClientConnection.GetDatabaseAccount = self.original_get_database_account
def test_retry_policy_does_not_mark_null_locations_unavailable(self): self.original_get_database_account = cosmos_client.CosmosClient.GetDatabaseAccount cosmos_client.CosmosClient.GetDatabaseAccount = self.mock_get_database_account client = cosmos_client.CosmosClient(self.DEFAULT_ENDPOINT, {'masterKey': self.MASTER_KEY}, None, documents.ConsistencyLevel.Eventual) endpoint_manager = global_endpoint_manager._GlobalEndpointManager(client) self.original_mark_endpoint_unavailable_for_read_function = endpoint_manager.mark_endpoint_unavailable_for_read endpoint_manager.mark_endpoint_unavailable_for_read = self._mock_mark_endpoint_unavailable_for_read self.original_mark_endpoint_unavailable_for_write_function = endpoint_manager.mark_endpoint_unavailable_for_write endpoint_manager.mark_endpoint_unavailable_for_write = self._mock_mark_endpoint_unavailable_for_write self.original_resolve_service_endpoint = endpoint_manager.resolve_service_endpoint endpoint_manager.resolve_service_endpoint = self._mock_resolve_service_endpoint # Read and write counters count the number of times the endpoint manager's # mark_endpoint_unavailable_for_read() and mark_endpoint_unavailable_for_read() # functions were called. When a 'None' location is returned by resolve_service_endpoint(), # these functions should not be called self._read_counter = 0 self._write_counter = 0 request = _RequestObject(http_constants.ResourceType.Document, documents._OperationType.Read) endpointDiscovery_retry_policy = endpoint_discovery_retry_policy._EndpointDiscoveryRetryPolicy(documents.ConnectionPolicy(), endpoint_manager, request) endpointDiscovery_retry_policy.ShouldRetry(errors.HTTPFailure(http_constants.StatusCodes.FORBIDDEN)) self.assertEqual(self._read_counter, 0) self.assertEqual(self._write_counter, 0) self._read_counter = 0 self._write_counter = 0 request = _RequestObject(http_constants.ResourceType.Document, documents._OperationType.Create) endpointDiscovery_retry_policy = endpoint_discovery_retry_policy._EndpointDiscoveryRetryPolicy(documents.ConnectionPolicy(), endpoint_manager, request) endpointDiscovery_retry_policy.ShouldRetry(errors.HTTPFailure(http_constants.StatusCodes.FORBIDDEN)) self.assertEqual(self._read_counter, 0) self.assertEqual(self._write_counter, 0) endpoint_manager.mark_endpoint_unavailable_for_read = self.original_mark_endpoint_unavailable_for_read_function endpoint_manager.mark_endpoint_unavailable_for_write = self.original_mark_endpoint_unavailable_for_write_function cosmos_client.CosmosClient.GetDatabaseAccount = self.original_get_database_account
def _Execute(client, global_endpoint_manager, function, *args, **kwargs): """Exectutes the function with passed parameters applying all retry policies :param object client: Document client instance :param object global_endpoint_manager: Instance of _GlobalEndpointManager class :param function function: Function to be called wrapped with retries :param (non-keyworded, variable number of arguments list) *args: :param (keyworded, variable number of arguments list) **kwargs: """ # instantiate all retry policies here to be applied for each request execution endpointDiscovery_retry_policy = endpoint_discovery_retry_policy._EndpointDiscoveryRetryPolicy(client.connection_policy, global_endpoint_manager, *args) resourceThrottle_retry_policy = resource_throttle_retry_policy._ResourceThrottleRetryPolicy(client.connection_policy.RetryOptions.MaxRetryAttemptCount, client.connection_policy.RetryOptions.FixedRetryIntervalInMilliseconds, client.connection_policy.RetryOptions.MaxWaitTimeInSeconds) defaultRetry_policy = default_retry_policy._DefaultRetryPolicy(*args) sessionRetry_policy = session_retry_policy._SessionRetryPolicy(client.connection_policy.EnableEndpointDiscovery, global_endpoint_manager, *args) while True: try: if args: result = _ExecuteFunction(function, global_endpoint_manager, *args, **kwargs) else: result = _ExecuteFunction(function, *args, **kwargs) if not client.last_response_headers: client.last_response_headers = {} # setting the throttle related response headers before returning the result client.last_response_headers[HttpHeaders.ThrottleRetryCount] = resourceThrottle_retry_policy.current_retry_attempt_count client.last_response_headers[HttpHeaders.ThrottleRetryWaitTimeInMs] = resourceThrottle_retry_policy.cummulative_wait_time_in_milliseconds return result except errors.HTTPFailure as e: retry_policy = None if (e.status_code == StatusCodes.FORBIDDEN and e.sub_status == SubStatusCodes.WRITE_FORBIDDEN): retry_policy = endpointDiscovery_retry_policy elif e.status_code == StatusCodes.TOO_MANY_REQUESTS: retry_policy = resourceThrottle_retry_policy elif e.status_code == StatusCodes.NOT_FOUND and e.sub_status and e.sub_status == SubStatusCodes.READ_SESSION_NOTAVAILABLE: retry_policy = sessionRetry_policy else: retry_policy = defaultRetry_policy # If none of the retry policies applies or there is no retry needed, set the throttle related response hedaers and # re-throw the exception back # arg[0] is the request. It needs to be modified for write forbidden exception if not (retry_policy.ShouldRetry(e)): if not client.last_response_headers: client.last_response_headers = {} client.last_response_headers[HttpHeaders.ThrottleRetryCount] = resourceThrottle_retry_policy.current_retry_attempt_count client.last_response_headers[HttpHeaders.ThrottleRetryWaitTimeInMs] = resourceThrottle_retry_policy.cummulative_wait_time_in_milliseconds if len(args) > 0 and args[0].should_clear_session_token_on_session_read_failure: client.session.clear_session_token(client.last_response_headers) raise else: # Wait for retry_after_in_milliseconds time before the next retry time.sleep(retry_policy.retry_after_in_milliseconds / 1000.0)