async def _send_async_helper(self, cdm_request: 'CdmHttpRequest', callback=None) -> 'CdmHttpResponse': """ Sends a CDM request with the retry logic helper function. :param cdm_request: The CDM Http request. :param callback: An optional parameter which specifies a callback function that gets executed after we try to execute the HTTP request. :return: The Cdm Http response. """ full_url = None # type : str if self._api_endpoint is not None: full_url = self._combine_urls(self._api_endpoint, cdm_request.requested_url) else: full_url = cdm_request.requested_url data = None # type: json # Set the content to be in the proper form and headers to denote # the content type. if cdm_request.content is not None: data = cdm_request.content cdm_request.headers['Content-Type'] = cdm_request.content_type request = urllib.request.Request(full_url, method=cdm_request.method, data=data) for key in cdm_request.headers: request.add_header(key, cdm_request.headers[key]) # If the number of retries is 0, we only try once, otherwise we retry the specified # number of times. for retry_number in range(cdm_request.number_of_retries + 1): cdm_response = None # type: CdmHttpResponse has_failed = False # type: bool try: # Send the request and convert timeout to seconds from milliseconds. with urllib.request.urlopen( request, timeout=cdm_request.timeout / 1000) as response: # type: http.client.HTTPResponse if response is not None: cdm_response = CdmHttpResponse() encoded_content = response.read() # Check whether we have appropriate attributes on the object. if hasattr(encoded_content, 'decode'): cdm_response.content = encoded_content.decode( 'utf-8') if hasattr(response, 'status'): cdm_response.reason = response.reason cdm_response.status_code = response.status # Successful requests have HTTP standard status codes in the 2xx form. cdm_response.is_successful = response.status // 100 == 2 if hasattr(response, 'getheaders'): cdm_response.response_headers = response.getheaders( ) except urllib.error.URLError: has_failed = True raise except Exception as exception: has_failed = True if callback is None or retry_number == cdm_request.number_of_retries: if retry_number != 0: raise CdmNumberOfRetriesExceededException else: if exception.args and exception.args[ 0].args and exception.args[0].args[ 0] == 'timed out': raise CdmTimedOutException else: raise exception # Check whether we have a callback function set and whether this is not our last retry. if callback is not None and retry_number != cdm_request.number_of_retries: # Call the callback function with the retry numbers starting from 1. wait_time = callback(cdm_response, has_failed, retry_number + 1) # type: int if wait_time is None: return cdm_response # Convert from milliseconds to seconds and wait the time specified by the callback. await asyncio.sleep(wait_time / 1000) else: # CDM Http Response exists, could be successful or bad (e.g. 403/404), it is up to caller to deal with it. if cdm_response is not None: return cdm_response if retry_number == 0: return None raise CdmNumberOfRetriesExceededException raise CdmNumberOfRetriesExceededException
async def _send_async_helper( self, cdm_request: 'CdmHttpRequest', callback=None, ctx: Optional['CdmCorpusContext'] = None) -> 'CdmHttpResponse': """ Sends a CDM request with the retry logic helper function. :param cdm_request: The CDM Http request. :param callback: An optional parameter which specifies a callback function that gets executed after we try to execute the HTTP request. :return: The Cdm Http response. """ full_url = None # type : str if self._api_endpoint is not None: full_url = self._combine_urls(self._api_endpoint, cdm_request.requested_url) else: full_url = cdm_request.requested_url data = None # type: json # Set the content to be in the proper form and headers to denote # the content type. if cdm_request.content is not None: data = cdm_request.content cdm_request.headers['Content-Type'] = cdm_request.content_type # urllib.request.Request() expects 'data' to be in bytes, so we convert to bytes here. if data is not None: data = data.encode("utf-8") request = urllib.request.Request(full_url, method=cdm_request.method, data=data) for key in cdm_request.headers: request.add_header(key, cdm_request.headers[key]) # If the number of retries is 0, we only try once, otherwise we retry the specified # number of times. for retry_number in range(cdm_request.number_of_retries + 1): cdm_response = None # type: CdmHttpResponse has_failed = False # type: bool try: start_time = datetime.now() if ctx is not None: logger.info( ctx, self._TAG, self._send_async_helper, None, 'Sending request: {}, request type: {}, request url: {}, retry number: {}.' .format(cdm_request.request_id, request.method, cdm_request._strip_sas_sig(), retry_number)) # Send the request and convert timeout to seconds from milliseconds. with (await in_thread_urlopen(request, timeout=cdm_request.timeout / 1000)) as response: if response is not None: end_time = datetime.now() if ctx is not None: logger.info( ctx, self._TAG, self._send_async_helper, None, 'Reponse for request {} received with elapsed time: {} ms.' .format( cdm_request.request_id, (end_time - start_time).total_seconds() * 1000.0)) cdm_response = CdmHttpResponse() encoded_content = response.read() # Check whether we have appropriate attributes on the object. if hasattr(encoded_content, 'decode'): cdm_response.content = encoded_content.decode( 'utf-8') if hasattr(response, 'status'): cdm_response.reason = response.reason cdm_response.status_code = response.status # Successful requests have HTTP standard status codes in the 2xx form. cdm_response.is_successful = response.status // 100 == 2 if hasattr(response, 'getheaders'): cdm_response.response_headers = dict( response.getheaders()) except urllib.error.URLError as exception: has_failed = True if callback is None or retry_number == cdm_request.number_of_retries: if retry_number != 0: raise CdmNumberOfRetriesExceededException(exception) else: if exception.args and exception.args[ 0].args and exception.args[0].args[ 0] == 'timed out': if ctx is not None: logger.info( ctx, self._TAG, self._send_async_helper, None, 'Reponse for request {} received with elapsed time: {} ms.' .format(cdm_request.request_id, (end_time - start_time).total_seconds() * 1000.0)) raise CdmTimedOutException('Request timeout.') else: raise exception except Exception as exception: has_failed = True raise # Check whether we have a callback function set and whether this is not our last retry. if callback is not None and retry_number != cdm_request.number_of_retries: # Call the callback function with the retry numbers starting from 1. wait_time = callback(cdm_response, has_failed, retry_number + 1) # type: int if wait_time is None: return cdm_response # Convert from milliseconds to seconds and wait the time specified by the callback. await asyncio.sleep(wait_time / 1000) else: # CDM Http Response exists, could be successful or bad (e.g. 403/404), it is up to caller to deal with it. if cdm_response is not None: return cdm_response if retry_number == 0: return None raise CdmNumberOfRetriesExceededException() raise CdmNumberOfRetriesExceededException()