def make_fetch_call(rpc, url, payload=None, method=GET, headers={},
allow_truncated=False, follow_redirects=True,
validate_certificate=None):
"""Executes the RPC call to fetch a given HTTP URL.
The first argument is a UserRPC instance. See urlfetch.fetch for a
thorough description of remaining arguments.
"""
assert rpc.service == 'urlfetch', repr(rpc.service)
if isinstance(method, basestring):
method = method.upper()
method = _URL_STRING_MAP.get(method, method)
if method not in _VALID_METHODS:
raise InvalidMethodError('Invalid method %s.' % str(method))
if _is_fetching_self(url, method):
raise InvalidURLError("App cannot fetch the same URL as the one used for "
"the request.")
request = urlfetch_service_pb.URLFetchRequest()
response = urlfetch_service_pb.URLFetchResponse()
request.set_url(url)
if method == GET:
request.set_method(urlfetch_service_pb.URLFetchRequest.GET)
elif method == POST:
request.set_method(urlfetch_service_pb.URLFetchRequest.POST)
elif method == HEAD:
request.set_method(urlfetch_service_pb.URLFetchRequest.HEAD)
elif method == PUT:
request.set_method(urlfetch_service_pb.URLFetchRequest.PUT)
elif method == DELETE:
request.set_method(urlfetch_service_pb.URLFetchRequest.DELETE)
if payload and (method == POST or method == PUT):
request.set_payload(payload)
for key, value in headers.iteritems():
header_proto = request.add_header()
header_proto.set_key(key)
header_proto.set_value(str(value))
request.set_followredirects(follow_redirects)
if validate_certificate is not None:
request.set_mustvalidateservercertificate(validate_certificate)
if rpc.deadline is not None:
request.set_deadline(rpc.deadline)
rpc.make_call('Fetch', request, response, _get_fetch_result, allow_truncated)
def test_exception(self):
urlfetch_request = urlfetch_service_pb.URLFetchRequest()
urlfetch_request.set_url('exception')
urlfetch_request.set_method(urlfetch_service_pb.URLFetchRequest.GET)
expected_remote_response = remote_api_pb.Response()
expected_remote_response.set_exception(
pickle.dumps(RuntimeError(repr(IOError('the remote error')))))
self._assert_remote_call(expected_remote_response, urlfetch_request,
'urlfetch', 'Fetch')
def test_application_error(self):
urlfetch_request = urlfetch_service_pb.URLFetchRequest()
urlfetch_request.set_url('application_error')
urlfetch_request.set_method(urlfetch_service_pb.URLFetchRequest.GET)
expected_remote_response = remote_api_pb.Response()
expected_remote_response.mutable_application_error().set_code(23)
expected_remote_response.mutable_application_error().set_detail('details')
expected_remote_response.set_exception(pickle.dumps(
apiproxy_errors.ApplicationError(23, 'details')))
self._assert_remote_call(
expected_remote_response, urlfetch_request, 'urlfetch', 'Fetch')
def make_fetch_call(rpc,
url,
payload=None,
method=GET,
headers={},
allow_truncated=False,
follow_redirects=True,
validate_certificate=None):
"""Executes the RPC call to fetch a given HTTP URL.
The first argument is a UserRPC instance. See `urlfetch.fetch` for a
thorough description of the remaining arguments.
Raises:
InvalidMethodError: If the requested method is not in `_VALID_METHODS`.
ResponseTooLargeError: If the response payload is too large.
InvalidURLError: If there are issues with the content or size of the
requested URL
Returns:
The RPC object that was passed into the function.
"""
assert rpc.service == 'urlfetch', repr(rpc.service)
if isinstance(method, basestring):
method = method.upper()
method = _URL_STRING_MAP.get(method, method)
if method not in _VALID_METHODS:
raise InvalidMethodError('Invalid method %s.' % str(method))
if _is_fetching_self(url, method):
raise InvalidURLError(
"App cannot fetch the same URL as the one used for "
"the request.")
request = urlfetch_service_pb.URLFetchRequest()
response = urlfetch_service_pb.URLFetchResponse()
if isinstance(url, unicode):
url = url.encode('UTF-8')
request.set_url(url)
if method == GET:
request.set_method(urlfetch_service_pb.URLFetchRequest.GET)
elif method == POST:
request.set_method(urlfetch_service_pb.URLFetchRequest.POST)
elif method == HEAD:
request.set_method(urlfetch_service_pb.URLFetchRequest.HEAD)
elif method == PUT:
request.set_method(urlfetch_service_pb.URLFetchRequest.PUT)
elif method == DELETE:
request.set_method(urlfetch_service_pb.URLFetchRequest.DELETE)
elif method == PATCH:
request.set_method(urlfetch_service_pb.URLFetchRequest.PATCH)
if payload and method in (POST, PUT, PATCH):
request.set_payload(payload)
for key, value in headers.iteritems():
header_proto = request.add_header()
header_proto.set_key(key)
header_proto.set_value(str(value))
request.set_followredirects(follow_redirects)
if validate_certificate is not None:
request.set_mustvalidateservercertificate(validate_certificate)
if rpc.deadline is not None:
request.set_deadline(rpc.deadline)
rpc.make_call('Fetch', request, response, _get_fetch_result,
allow_truncated)
return rpc
def fetch(url, payload=None, method=GET, headers={}, allow_truncated=False):
"""Fetches the given HTTP URL, blocking until the result is returned.
Other optional parameters are:
method: GET, POST, HEAD, PUT, or DELETE
payload: POST or PUT payload (implies method is not GET, HEAD, or DELETE)
headers: dictionary of HTTP headers to send with the request
allow_truncated: if true, truncate large responses and return them without
error. otherwise, ResponseTooLargeError will be thrown when a response is
truncated.
We use a HTTP/1.1 compliant proxy to fetch the result.
The returned data structure has the following fields:
content: string containing the response from the server
status_code: HTTP status code returned by the server
headers: dictionary of headers returned by the server
If the URL is an empty string or obviously invalid, we throw an
urlfetch.InvalidURLError. If the server cannot be contacted, we throw a
urlfetch.DownloadError. Note that HTTP errors are returned as a part
of the returned structure, so HTTP errors like 404 do not result in an
exception.
"""
request = urlfetch_service_pb.URLFetchRequest()
response = urlfetch_service_pb.URLFetchResponse()
request.set_url(url)
if method == GET:
request.set_method(urlfetch_service_pb.URLFetchRequest.GET)
elif method == POST:
request.set_method(urlfetch_service_pb.URLFetchRequest.POST)
elif method == HEAD:
request.set_method(urlfetch_service_pb.URLFetchRequest.HEAD)
elif method == PUT:
request.set_method(urlfetch_service_pb.URLFetchRequest.PUT)
elif method == DELETE:
request.set_method(urlfetch_service_pb.URLFetchRequest.DELETE)
if payload and (method == POST or method == PUT):
request.set_payload(payload)
for key, value in headers.iteritems():
header_proto = request.add_header()
header_proto.set_key(key)
header_proto.set_value(value)
try:
apiproxy_stub_map.MakeSyncCall('urlfetch', 'Fetch', request, response)
except apiproxy_errors.ApplicationError, e:
if (e.application_error ==
urlfetch_service_pb.URLFetchServiceError.INVALID_URL):
raise InvalidURLError()
if (e.application_error ==
urlfetch_service_pb.URLFetchServiceError.UNSPECIFIED_ERROR):
raise DownloadError()
if (e.application_error ==
urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR):
raise DownloadError()
if (e.application_error ==
urlfetch_service_pb.URLFetchServiceError.RESPONSE_TOO_LARGE):
raise ResponseTooLargeError(None)
raise e
def fetch(url,
payload=None,
method=GET,
headers={},
allow_truncated=False,
follow_redirects=True):
"""Fetches the given HTTP URL, blocking until the result is returned.
Other optional parameters are:
method: GET, POST, HEAD, PUT, or DELETE
payload: POST or PUT payload (implies method is not GET, HEAD, or DELETE)
headers: dictionary of HTTP headers to send with the request
allow_truncated: if true, truncate large responses and return them without
error. otherwise, ResponseTooLargeError will be thrown when a response is
truncated.
follow_redirects: if true (the default), redirects are
transparently followed and the response (if less than 5
redirects) contains the final destination's payload and the
response status is 200. You lose, however, the redirect chain
information. If false, you see the HTTP response yourself,
including the 'Location' header, and redirects are not
followed.
We use a HTTP/1.1 compliant proxy to fetch the result.
The returned data structure has the following fields:
content: string containing the response from the server
status_code: HTTP status code returned by the server
headers: dictionary of headers returned by the server
If the URL is an empty string or obviously invalid, we throw an
urlfetch.InvalidURLError. If the server cannot be contacted, we throw a
urlfetch.DownloadError. Note that HTTP errors are returned as a part
of the returned structure, so HTTP errors like 404 do not result in an
exception.
"""
if isinstance(method, basestring):
method = method.upper()
method = _URL_STRING_MAP.get(method, method)
if method not in _VALID_METHODS:
raise InvalidMethodError('Invalid method %s.' % str(method))
if _is_fetching_self(url, method):
raise InvalidURLError(
"App cannot fetch the same URL as the one used for "
"the request.")
request = urlfetch_service_pb.URLFetchRequest()
response = urlfetch_service_pb.URLFetchResponse()
request.set_url(url)
if method == GET:
request.set_method(urlfetch_service_pb.URLFetchRequest.GET)
elif method == POST:
request.set_method(urlfetch_service_pb.URLFetchRequest.POST)
elif method == HEAD:
request.set_method(urlfetch_service_pb.URLFetchRequest.HEAD)
elif method == PUT:
request.set_method(urlfetch_service_pb.URLFetchRequest.PUT)
elif method == DELETE:
request.set_method(urlfetch_service_pb.URLFetchRequest.DELETE)
if payload and (method == POST or method == PUT):
request.set_payload(payload)
for key, value in headers.iteritems():
header_proto = request.add_header()
header_proto.set_key(key)
header_proto.set_value(str(value))
request.set_followredirects(follow_redirects)
try:
apiproxy_stub_map.MakeSyncCall('urlfetch', 'Fetch', request, response)
except apiproxy_errors.ApplicationError, e:
if (e.application_error ==
urlfetch_service_pb.URLFetchServiceError.INVALID_URL):
raise InvalidURLError(str(e))
if (e.application_error ==
urlfetch_service_pb.URLFetchServiceError.UNSPECIFIED_ERROR):
raise DownloadError(str(e))
if (e.application_error ==
urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR):
raise DownloadError(str(e))
if (e.application_error ==
urlfetch_service_pb.URLFetchServiceError.RESPONSE_TOO_LARGE):
raise ResponseTooLargeError(None)
if (e.application_error ==
urlfetch_service_pb.URLFetchServiceError.DEADLINE_EXCEEDED):
raise DownloadError(str(e))
raise e
def fetch(url, payload=None, method=urlfetch.GET, headers={},
allow_truncated=False, follow_redirects=True,
callback=None, async_proxy=None,
deadline=5):
"""Fetches the given HTTP URL, blocking until the result is returned.
Other optional parameters are:
method: GET, POST, HEAD, PUT, or DELETE
payload: POST or PUT payload (implies method is not GET, HEAD, or DELETE)
headers: dictionary of HTTP headers to send with the request
allow_truncated: if true, truncate large responses and return them without
error. otherwise, ResponseTooLargeError will be thrown when a response is
truncated.
follow_redirects: Whether or not redirects should be followed.
callback: Callable that takes (_URLFetchResult, URLFetchException).
Exactly one of the two arguments is None. Required if async_proxy is
not None.
async_proxy: If not None, instance of AsyncAPIProxy to use for executing
asynchronous API calls.
deadline: How long to allow the request to wait, in seconds. Defaults
to 5 seconds.
We use a HTTP/1.1 compliant proxy to fetch the result.
The returned data structure has the following fields:
content: string containing the response from the server
status_code: HTTP status code returned by the server
headers: dictionary of headers returned by the server
If the URL is an empty string or obviously invalid, we throw an
urlfetch.InvalidURLError. If the server cannot be contacted, we throw a
urlfetch.DownloadError. Note that HTTP errors are returned as a part
of the returned structure, so HTTP errors like 404 do not result in an
exception.
"""
request = urlfetch_service_pb.URLFetchRequest()
response = urlfetch_service_pb.URLFetchResponse()
request.set_url(url)
if isinstance(method, basestring):
method = method.upper()
method = urlfetch._URL_STRING_MAP.get(method, method)
if method not in urlfetch._VALID_METHODS:
raise InvalidMethodError('Invalid method %s.' % str(method))
if method == urlfetch.GET:
request.set_method(urlfetch_service_pb.URLFetchRequest.GET)
elif method == urlfetch.POST:
request.set_method(urlfetch_service_pb.URLFetchRequest.POST)
elif method == urlfetch.HEAD:
request.set_method(urlfetch_service_pb.URLFetchRequest.HEAD)
elif method == urlfetch.PUT:
request.set_method(urlfetch_service_pb.URLFetchRequest.PUT)
elif method == urlfetch.DELETE:
request.set_method(urlfetch_service_pb.URLFetchRequest.DELETE)
request.set_followredirects(follow_redirects)
if payload and (method == urlfetch.POST or method == urlfetch.PUT):
request.set_payload(payload)
for key, value in headers.iteritems():
header_proto = request.add_header()
header_proto.set_key(key)
header_proto.set_value(value)
if async_proxy:
def completion_callback(response, urlfetch_exception):
result, user_exception = HandleResult(response, urlfetch_exception,
allow_truncated)
callback(result, user_exception)
async_proxy.start_call('urlfetch', 'Fetch', request, response,
completion_callback, deadline=deadline)
return
user_exception = None
try:
apiproxy_stub_map.MakeSyncCall('urlfetch', 'Fetch', request, response)
except apiproxy_errors.ApplicationError, e:
user_exception = e
