def _to_url(cls, url_path): try: return u'{0}{1}'.format(cls.BASE_URL, url_path) except Exception as e: write_error_message(url_path) write_exception(e) raise e
def _wait_for_response(self, requests, to_json): """Issue a batch of requests and wait for the responses. Args: requests - A list of requests to_json - A boolean, should the responses be returned as JSON blobs Returns: A list of dicts if to_json, a list of grequest.response otherwise """ all_responses = [] for request, response in zip(requests, grequests.map(requests)): if not response: response = MultiRequest._FakeResponse(request, '<UNKNOWN>') if 200 != response.status_code: write_error_message('url[{0}] status_code[{1}]'.format( response.request.url, response.status_code)) if to_json: all_responses.append(response.json()) else: all_responses.append(response) return all_responses
def test_write_error_message(self): message = 'Look for me in validation' expected = '[ERROR] Look for me in validation\n' write_error_message(message) output = self._stringio.getvalue() T.assert_equal(output, expected)
def _wait_for_response(self, requests, to_json): """Issue a batch of requests and wait for the responses. Args: requests - A list of requests to_json - A boolean, should the responses be returned as JSON blobs Returns: A list of dicts if to_json, a list of grequest.response otherwise """ all_responses = [] for retry in range(self._max_retry): try: responses = grequests.map(requests) valid_responses = [ response for response in responses if response ] failed_auth_responses = [ response for response in responses if response.status_code == 403 ] if failed_auth_responses: raise ConnectionError( 'Credentials not authorized to access URL') if len(valid_responses) != len(requests): continue else: break except ConnectionError: raise except: pass if retry == self._max_retry: raise ConnectionError( 'Unable to complete batch of requests within max_retry retries' ) for request, response in zip(requests, responses): if not response: # should have caught this earlier, but if not ... raise ConnectionError( 'Request to {0} had an empty response'.format(request.url)) if 200 != response.status_code: write_error_message('url[{0}] status_code[{1}]'.format( response.request.url, response.status_code)) if to_json: all_responses.append(response.json()) else: all_responses.append(response) return all_responses
def _to_urls(cls, fmt_url_path, url_path_args): url_paths = [] for path_arg in url_path_args: try: url_paths.append(fmt_url_path.format(path_arg)) except Exception as e: write_error_message(path_arg) write_exception(e) raise e return [cls._to_url(url_path) for url_path in url_paths]
def wrapper(*args, **kwargs): try: result = fn(*args, **kwargs) return result except InvalidRequestError as e: write_exception(e) if hasattr(e, 'request'): write_error_message('request {0}'.format(repr(e.request))) if hasattr(e, 'response'): write_error_message('response {0}'.format(repr(e.response))) raise e
def _wait_for_response(self, requests, to_json): """Issue a batch of requests and wait for the responses. Args: requests - A list of requests to_json - A boolean, should the responses be returned as JSON blobs Returns: A list of dicts if to_json, a list of grequest.response otherwise """ all_responses = [] for retry in range(self._max_retry): try: logging.debug('Try #{0}'.format(retry + 1)) # TODO retry only for the responses that have not finished successfully yet responses = grequests.map(requests, self._handle_exception) valid_responses = [response for response in responses if response] if any(response is not None and response.status_code == 403 for response in responses): raise InvalidRequestError('Access forbidden') if len(valid_responses) == len(requests): break logging.warning('Try #{0}. Expected {1} valid response(s) but only got {2}.'.format( retry + 1, len(requests), len(valid_responses))) if retry + 1 == self._max_retry: raise InvalidRequestError('Unable to complete batch of requests within {0} retries'.format(self._max_retry)) except InvalidRequestError: raise except Exception as e: # log the exception for the informative purposes and pass to the next iteration logging.exception('Try #{0}. Exception occured: {1}. Retrying.'.format(retry + 1, e)) pass for request, response in zip(requests, responses): if 200 != response.status_code: write_error_message('url[{0}] status_code[{1}]'.format(response.request.url, response.status_code)) if to_json: try: all_responses.append(response.json()) except ValueError: raise InvalidRequestError('Expected JSON response from: {0}'.format(response.request.url)) else: all_responses.append(response) return all_responses
def _wait_for_response(self, requests, to_json): """Issue a batch of requests and wait for the responses. Args: requests - A list of requests to_json - A boolean, should the responses be returned as JSON blobs Returns: A list of dicts if to_json, a list of grequest.response otherwise """ all_responses = [] for retry in range(self._max_retry): try: responses = grequests.map(requests) valid_responses = [response for response in responses if response] failed_auth_responses = [response for response in responses if response.status_code == 403] if failed_auth_responses: raise ConnectionError('Credentials not authorized to access URL') if len(valid_responses) != len(requests): continue else: break except ConnectionError: raise except: pass if retry == self._max_retry: raise ConnectionError('Unable to complete batch of requests within max_retry retries') for request, response in zip(requests, responses): if not response: # should have caught this earlier, but if not ... raise ConnectionError('Request to {0} had an empty response'.format(request.url)) if 200 != response.status_code: write_error_message('url[{0}] status_code[{1}]'.format(response.request.url, response.status_code)) if to_json: all_responses.append(response.json()) else: all_responses.append(response) return all_responses
def _wait_for_response(self, requests, to_json): """Issue a batch of requests and wait for the responses. Args: requests - A list of requests to_json - A boolean, should the responses be returned as JSON blobs Returns: A list of dicts if to_json, a list of grequest.response otherwise """ all_responses = [] for request, response in zip(requests, grequests.map(requests)): if not response: response = MultiRequest._FakeResponse(request, '<UNKNOWN>') if 200 != response.status_code: write_error_message('url[{0}] status_code[{1}]'.format(response.request.url, response.status_code)) if to_json: all_responses.append(response.json()) else: all_responses.append(response) return all_responses