def data(self):
if not hasattr(self, '_data'):
request = URLRequest(self.url)
# Look in the cache for etag / last modified headers to use
# TODO: "expires" header could be supported
if self.env and self.env.cache:
headers = self.env.cache.get(('url', 'headers', self.url))
if headers:
etag, lmod = headers
if etag: request.add_header('If-None-Match', etag)
if lmod: request.add_header('If-Modified-Since', lmod)
# Make a request
try:
response = urlopen(request)
except HTTPError as e:
if e.code != 304:
raise
# Use the cached version of the url
self._data = self.env.cache.get(('url', 'contents', self.url))
else:
with contextlib.closing(response):
self._data = response.read()
# Cache the info from this request
if self.env and self.env.cache:
self.env.cache.set(('url', 'headers', self.url),
(response.headers.get("ETag"),
response.headers.get("Last-Modified")))
self.env.cache.set(('url', 'contents', self.url),
self._data)
return self._data
def open_page(url: str,
ignore_client_error: bool = False,
accept_header: str = '*/*') -> addinfourl:
"""Open a connection to retrieve a resource via HTTP GET.
@param url:
The URL of the resource to request.
@param ignore_client_error:
If C{True}, a client error (HTTP status 400) is not reported
as an error. This is useful to avoid false positives when
making speculative requests.
@param accept_header:
HTTP C{Accept} header to use for the request.
@return:
A response object that contains an open stream that data can
be read from.
@raise apetest.report.FetchFailure:
If no connection could be opened.
"""
# TODO: Figure out how to do authentication, "user:password@" in
# the URL does not work.
# There is support for HTTP basic auth in urllib.
url_req = URLRequest(url)
url_req.add_header('Accept', accept_header)
url_req.add_header('User-Agent', USER_AGENT)
while True:
try:
response: addinfourl = _URL_OPENER.open(url_req)
return response
except HTTPError as ex:
if ex.code == 503:
if 'retry-after' in ex.headers:
try:
seconds = int(ex.headers['retry-after'])
except ValueError:
# TODO: HTTP spec allows a date string here.
_LOG.warning('Parsing of "Retry-After" dates '
'is not yet implemented')
seconds = 5
else:
seconds = 5
_LOG.info(
'Server not ready yet, trying again '
'in %d seconds', seconds)
sleep(seconds)
elif 300 <= ex.code < 400:
# Do not treat redirects as errors.
return ex
elif ex.code == 400 and ignore_client_error:
# Ignore generic client error, because we used a speculative
# request and 400 can be the correct response to that.
return ex
else:
message = f'HTTP error {ex.code:d}: {ex.reason}'
raise FetchFailure(url, message, http_error=ex)
except URLError as ex:
raise FetchFailure(url, str(ex.reason))
except OSError as ex:
raise FetchFailure(url, ex.strerror)
def auth_request(req: str, payload: dict, error: bool = True) -> (int, dict):
req_url = AUTHSERVER_URL.format(req)
data = json.dumps(payload).encode("ascii")
req = URLRequest(req_url,
data,
headers={
"Content-Type": "application/json",
"Content-Length": len(data)
},
method="POST")
try:
res = urlreq.urlopen(req) # type: HTTPResponse
except HTTPError as err:
res = cast(HTTPResponse, err.fp)
try:
res_data = json.load(res)
except JSONDecodeError:
res_data = {}
if error and res.status != 200:
raise AuthError(res_data["errorMessage"])
return res.status, res_data
def on_request(self, request):
try:
# Extract information from request
query = request.payload.decode(encoding="UTF-8")
logger.info("Service received request payload: " + query)
# Send HTTP request to OpenWeatherMap
req = URLRequest(CURRENT_WEATHER_URL.format(query, API_KEY),
None, {'Content-Type': 'text/json'})
f = urlopen(req)
weather_response = f.read()
f.close()
# Create the response message
response = Response(request)
# Populate the response payload
response.payload = weather_response
# Send the response
client.send_response(response)
except Exception as ex:
print(str(ex))
# Send error response
client.send_response(
ErrorResponse(
request,
error_message=str(ex).encode(encoding="UTF-8")))