def make_fetch_call(rpc, url, payload=None, method=GET, headers={}, allow_truncated=False, follow_redirects=True, validate_certificate=None): """Executes the RPC call to fetch a given HTTP URL. The first argument is a UserRPC instance. See urlfetch.fetch for a thorough description of remaining arguments. """ assert rpc.service == 'urlfetch', repr(rpc.service) if isinstance(method, basestring): method = method.upper() method = _URL_STRING_MAP.get(method, method) if method not in _VALID_METHODS: raise InvalidMethodError('Invalid method %s.' % str(method)) if _is_fetching_self(url, method): raise InvalidURLError("App cannot fetch the same URL as the one used for " "the request.") request = urlfetch_service_pb.URLFetchRequest() response = urlfetch_service_pb.URLFetchResponse() request.set_url(url) if method == GET: request.set_method(urlfetch_service_pb.URLFetchRequest.GET) elif method == POST: request.set_method(urlfetch_service_pb.URLFetchRequest.POST) elif method == HEAD: request.set_method(urlfetch_service_pb.URLFetchRequest.HEAD) elif method == PUT: request.set_method(urlfetch_service_pb.URLFetchRequest.PUT) elif method == DELETE: request.set_method(urlfetch_service_pb.URLFetchRequest.DELETE) if payload and (method == POST or method == PUT): request.set_payload(payload) for key, value in headers.iteritems(): header_proto = request.add_header() header_proto.set_key(key) header_proto.set_value(str(value)) request.set_followredirects(follow_redirects) if validate_certificate is not None: request.set_mustvalidateservercertificate(validate_certificate) if rpc.deadline is not None: request.set_deadline(rpc.deadline) rpc.make_call('Fetch', request, response, _get_fetch_result, allow_truncated)
def test_exception(self): urlfetch_request = urlfetch_service_pb.URLFetchRequest() urlfetch_request.set_url('exception') urlfetch_request.set_method(urlfetch_service_pb.URLFetchRequest.GET) expected_remote_response = remote_api_pb.Response() expected_remote_response.set_exception( pickle.dumps(RuntimeError(repr(IOError('the remote error'))))) self._assert_remote_call(expected_remote_response, urlfetch_request, 'urlfetch', 'Fetch')
def test_application_error(self): urlfetch_request = urlfetch_service_pb.URLFetchRequest() urlfetch_request.set_url('application_error') urlfetch_request.set_method(urlfetch_service_pb.URLFetchRequest.GET) expected_remote_response = remote_api_pb.Response() expected_remote_response.mutable_application_error().set_code(23) expected_remote_response.mutable_application_error().set_detail('details') expected_remote_response.set_exception(pickle.dumps( apiproxy_errors.ApplicationError(23, 'details'))) self._assert_remote_call( expected_remote_response, urlfetch_request, 'urlfetch', 'Fetch')
def make_fetch_call(rpc, url, payload=None, method=GET, headers={}, allow_truncated=False, follow_redirects=True, validate_certificate=None): """Executes the RPC call to fetch a given HTTP URL. The first argument is a UserRPC instance. See `urlfetch.fetch` for a thorough description of the remaining arguments. Raises: InvalidMethodError: If the requested method is not in `_VALID_METHODS`. ResponseTooLargeError: If the response payload is too large. InvalidURLError: If there are issues with the content or size of the requested URL Returns: The RPC object that was passed into the function. """ assert rpc.service == 'urlfetch', repr(rpc.service) if isinstance(method, basestring): method = method.upper() method = _URL_STRING_MAP.get(method, method) if method not in _VALID_METHODS: raise InvalidMethodError('Invalid method %s.' % str(method)) if _is_fetching_self(url, method): raise InvalidURLError( "App cannot fetch the same URL as the one used for " "the request.") request = urlfetch_service_pb.URLFetchRequest() response = urlfetch_service_pb.URLFetchResponse() if isinstance(url, unicode): url = url.encode('UTF-8') request.set_url(url) if method == GET: request.set_method(urlfetch_service_pb.URLFetchRequest.GET) elif method == POST: request.set_method(urlfetch_service_pb.URLFetchRequest.POST) elif method == HEAD: request.set_method(urlfetch_service_pb.URLFetchRequest.HEAD) elif method == PUT: request.set_method(urlfetch_service_pb.URLFetchRequest.PUT) elif method == DELETE: request.set_method(urlfetch_service_pb.URLFetchRequest.DELETE) elif method == PATCH: request.set_method(urlfetch_service_pb.URLFetchRequest.PATCH) if payload and method in (POST, PUT, PATCH): request.set_payload(payload) for key, value in headers.iteritems(): header_proto = request.add_header() header_proto.set_key(key) header_proto.set_value(str(value)) request.set_followredirects(follow_redirects) if validate_certificate is not None: request.set_mustvalidateservercertificate(validate_certificate) if rpc.deadline is not None: request.set_deadline(rpc.deadline) rpc.make_call('Fetch', request, response, _get_fetch_result, allow_truncated) return rpc
def fetch(url, payload=None, method=GET, headers={}, allow_truncated=False): """Fetches the given HTTP URL, blocking until the result is returned. Other optional parameters are: method: GET, POST, HEAD, PUT, or DELETE payload: POST or PUT payload (implies method is not GET, HEAD, or DELETE) headers: dictionary of HTTP headers to send with the request allow_truncated: if true, truncate large responses and return them without error. otherwise, ResponseTooLargeError will be thrown when a response is truncated. We use a HTTP/1.1 compliant proxy to fetch the result. The returned data structure has the following fields: content: string containing the response from the server status_code: HTTP status code returned by the server headers: dictionary of headers returned by the server If the URL is an empty string or obviously invalid, we throw an urlfetch.InvalidURLError. If the server cannot be contacted, we throw a urlfetch.DownloadError. Note that HTTP errors are returned as a part of the returned structure, so HTTP errors like 404 do not result in an exception. """ request = urlfetch_service_pb.URLFetchRequest() response = urlfetch_service_pb.URLFetchResponse() request.set_url(url) if method == GET: request.set_method(urlfetch_service_pb.URLFetchRequest.GET) elif method == POST: request.set_method(urlfetch_service_pb.URLFetchRequest.POST) elif method == HEAD: request.set_method(urlfetch_service_pb.URLFetchRequest.HEAD) elif method == PUT: request.set_method(urlfetch_service_pb.URLFetchRequest.PUT) elif method == DELETE: request.set_method(urlfetch_service_pb.URLFetchRequest.DELETE) if payload and (method == POST or method == PUT): request.set_payload(payload) for key, value in headers.iteritems(): header_proto = request.add_header() header_proto.set_key(key) header_proto.set_value(value) try: apiproxy_stub_map.MakeSyncCall('urlfetch', 'Fetch', request, response) except apiproxy_errors.ApplicationError, e: if (e.application_error == urlfetch_service_pb.URLFetchServiceError.INVALID_URL): raise InvalidURLError() if (e.application_error == urlfetch_service_pb.URLFetchServiceError.UNSPECIFIED_ERROR): raise DownloadError() if (e.application_error == urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR): raise DownloadError() if (e.application_error == urlfetch_service_pb.URLFetchServiceError.RESPONSE_TOO_LARGE): raise ResponseTooLargeError(None) raise e
def fetch(url, payload=None, method=GET, headers={}, allow_truncated=False, follow_redirects=True): """Fetches the given HTTP URL, blocking until the result is returned. Other optional parameters are: method: GET, POST, HEAD, PUT, or DELETE payload: POST or PUT payload (implies method is not GET, HEAD, or DELETE) headers: dictionary of HTTP headers to send with the request allow_truncated: if true, truncate large responses and return them without error. otherwise, ResponseTooLargeError will be thrown when a response is truncated. follow_redirects: if true (the default), redirects are transparently followed and the response (if less than 5 redirects) contains the final destination's payload and the response status is 200. You lose, however, the redirect chain information. If false, you see the HTTP response yourself, including the 'Location' header, and redirects are not followed. We use a HTTP/1.1 compliant proxy to fetch the result. The returned data structure has the following fields: content: string containing the response from the server status_code: HTTP status code returned by the server headers: dictionary of headers returned by the server If the URL is an empty string or obviously invalid, we throw an urlfetch.InvalidURLError. If the server cannot be contacted, we throw a urlfetch.DownloadError. Note that HTTP errors are returned as a part of the returned structure, so HTTP errors like 404 do not result in an exception. """ if isinstance(method, basestring): method = method.upper() method = _URL_STRING_MAP.get(method, method) if method not in _VALID_METHODS: raise InvalidMethodError('Invalid method %s.' % str(method)) if _is_fetching_self(url, method): raise InvalidURLError( "App cannot fetch the same URL as the one used for " "the request.") request = urlfetch_service_pb.URLFetchRequest() response = urlfetch_service_pb.URLFetchResponse() request.set_url(url) if method == GET: request.set_method(urlfetch_service_pb.URLFetchRequest.GET) elif method == POST: request.set_method(urlfetch_service_pb.URLFetchRequest.POST) elif method == HEAD: request.set_method(urlfetch_service_pb.URLFetchRequest.HEAD) elif method == PUT: request.set_method(urlfetch_service_pb.URLFetchRequest.PUT) elif method == DELETE: request.set_method(urlfetch_service_pb.URLFetchRequest.DELETE) if payload and (method == POST or method == PUT): request.set_payload(payload) for key, value in headers.iteritems(): header_proto = request.add_header() header_proto.set_key(key) header_proto.set_value(str(value)) request.set_followredirects(follow_redirects) try: apiproxy_stub_map.MakeSyncCall('urlfetch', 'Fetch', request, response) except apiproxy_errors.ApplicationError, e: if (e.application_error == urlfetch_service_pb.URLFetchServiceError.INVALID_URL): raise InvalidURLError(str(e)) if (e.application_error == urlfetch_service_pb.URLFetchServiceError.UNSPECIFIED_ERROR): raise DownloadError(str(e)) if (e.application_error == urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR): raise DownloadError(str(e)) if (e.application_error == urlfetch_service_pb.URLFetchServiceError.RESPONSE_TOO_LARGE): raise ResponseTooLargeError(None) if (e.application_error == urlfetch_service_pb.URLFetchServiceError.DEADLINE_EXCEEDED): raise DownloadError(str(e)) raise e
def fetch(url, payload=None, method=urlfetch.GET, headers={}, allow_truncated=False, follow_redirects=True, callback=None, async_proxy=None, deadline=5): """Fetches the given HTTP URL, blocking until the result is returned. Other optional parameters are: method: GET, POST, HEAD, PUT, or DELETE payload: POST or PUT payload (implies method is not GET, HEAD, or DELETE) headers: dictionary of HTTP headers to send with the request allow_truncated: if true, truncate large responses and return them without error. otherwise, ResponseTooLargeError will be thrown when a response is truncated. follow_redirects: Whether or not redirects should be followed. callback: Callable that takes (_URLFetchResult, URLFetchException). Exactly one of the two arguments is None. Required if async_proxy is not None. async_proxy: If not None, instance of AsyncAPIProxy to use for executing asynchronous API calls. deadline: How long to allow the request to wait, in seconds. Defaults to 5 seconds. We use a HTTP/1.1 compliant proxy to fetch the result. The returned data structure has the following fields: content: string containing the response from the server status_code: HTTP status code returned by the server headers: dictionary of headers returned by the server If the URL is an empty string or obviously invalid, we throw an urlfetch.InvalidURLError. If the server cannot be contacted, we throw a urlfetch.DownloadError. Note that HTTP errors are returned as a part of the returned structure, so HTTP errors like 404 do not result in an exception. """ request = urlfetch_service_pb.URLFetchRequest() response = urlfetch_service_pb.URLFetchResponse() request.set_url(url) if isinstance(method, basestring): method = method.upper() method = urlfetch._URL_STRING_MAP.get(method, method) if method not in urlfetch._VALID_METHODS: raise InvalidMethodError('Invalid method %s.' % str(method)) if method == urlfetch.GET: request.set_method(urlfetch_service_pb.URLFetchRequest.GET) elif method == urlfetch.POST: request.set_method(urlfetch_service_pb.URLFetchRequest.POST) elif method == urlfetch.HEAD: request.set_method(urlfetch_service_pb.URLFetchRequest.HEAD) elif method == urlfetch.PUT: request.set_method(urlfetch_service_pb.URLFetchRequest.PUT) elif method == urlfetch.DELETE: request.set_method(urlfetch_service_pb.URLFetchRequest.DELETE) request.set_followredirects(follow_redirects) if payload and (method == urlfetch.POST or method == urlfetch.PUT): request.set_payload(payload) for key, value in headers.iteritems(): header_proto = request.add_header() header_proto.set_key(key) header_proto.set_value(value) if async_proxy: def completion_callback(response, urlfetch_exception): result, user_exception = HandleResult(response, urlfetch_exception, allow_truncated) callback(result, user_exception) async_proxy.start_call('urlfetch', 'Fetch', request, response, completion_callback, deadline=deadline) return user_exception = None try: apiproxy_stub_map.MakeSyncCall('urlfetch', 'Fetch', request, response) except apiproxy_errors.ApplicationError, e: user_exception = e