def data(self): if not hasattr(self, '_data'): request = URLRequest(self.url) # Look in the cache for etag / last modified headers to use # TODO: "expires" header could be supported if self.env and self.env.cache: headers = self.env.cache.get(('url', 'headers', self.url)) if headers: etag, lmod = headers if etag: request.add_header('If-None-Match', etag) if lmod: request.add_header('If-Modified-Since', lmod) # Make a request try: response = urlopen(request) except HTTPError as e: if e.code != 304: raise # Use the cached version of the url self._data = self.env.cache.get(('url', 'contents', self.url)) else: with contextlib.closing(response): self._data = response.read() # Cache the info from this request if self.env and self.env.cache: self.env.cache.set(('url', 'headers', self.url), (response.headers.get("ETag"), response.headers.get("Last-Modified"))) self.env.cache.set(('url', 'contents', self.url), self._data) return self._data
def open_page(url: str, ignore_client_error: bool = False, accept_header: str = '*/*') -> addinfourl: """Open a connection to retrieve a resource via HTTP GET. @param url: The URL of the resource to request. @param ignore_client_error: If C{True}, a client error (HTTP status 400) is not reported as an error. This is useful to avoid false positives when making speculative requests. @param accept_header: HTTP C{Accept} header to use for the request. @return: A response object that contains an open stream that data can be read from. @raise apetest.report.FetchFailure: If no connection could be opened. """ # TODO: Figure out how to do authentication, "user:password@" in # the URL does not work. # There is support for HTTP basic auth in urllib. url_req = URLRequest(url) url_req.add_header('Accept', accept_header) url_req.add_header('User-Agent', USER_AGENT) while True: try: response: addinfourl = _URL_OPENER.open(url_req) return response except HTTPError as ex: if ex.code == 503: if 'retry-after' in ex.headers: try: seconds = int(ex.headers['retry-after']) except ValueError: # TODO: HTTP spec allows a date string here. _LOG.warning('Parsing of "Retry-After" dates ' 'is not yet implemented') seconds = 5 else: seconds = 5 _LOG.info( 'Server not ready yet, trying again ' 'in %d seconds', seconds) sleep(seconds) elif 300 <= ex.code < 400: # Do not treat redirects as errors. return ex elif ex.code == 400 and ignore_client_error: # Ignore generic client error, because we used a speculative # request and 400 can be the correct response to that. return ex else: message = f'HTTP error {ex.code:d}: {ex.reason}' raise FetchFailure(url, message, http_error=ex) except URLError as ex: raise FetchFailure(url, str(ex.reason)) except OSError as ex: raise FetchFailure(url, ex.strerror)
def auth_request(req: str, payload: dict, error: bool = True) -> (int, dict): req_url = AUTHSERVER_URL.format(req) data = json.dumps(payload).encode("ascii") req = URLRequest(req_url, data, headers={ "Content-Type": "application/json", "Content-Length": len(data) }, method="POST") try: res = urlreq.urlopen(req) # type: HTTPResponse except HTTPError as err: res = cast(HTTPResponse, err.fp) try: res_data = json.load(res) except JSONDecodeError: res_data = {} if error and res.status != 200: raise AuthError(res_data["errorMessage"]) return res.status, res_data
def on_request(self, request): try: # Extract information from request query = request.payload.decode(encoding="UTF-8") logger.info("Service received request payload: " + query) # Send HTTP request to OpenWeatherMap req = URLRequest(CURRENT_WEATHER_URL.format(query, API_KEY), None, {'Content-Type': 'text/json'}) f = urlopen(req) weather_response = f.read() f.close() # Create the response message response = Response(request) # Populate the response payload response.payload = weather_response # Send the response client.send_response(response) except Exception as ex: print(str(ex)) # Send error response client.send_response( ErrorResponse( request, error_message=str(ex).encode(encoding="UTF-8")))