def test_error_with_response(self): resp = HTTPResponse(HTTPRequest('http://example.com/'), 403) with self.assertRaises(HTTPError) as cm: resp.rethrow() e = cm.exception self.assertEqual(str(e), "HTTP 403: Forbidden") self.assertEqual(repr(e), "HTTP 403: Forbidden")
def is_member(request): urlinfo = urlparse(request.url) group, uid = member_regex.match(urlinfo.path).group(1, 2) uname = list(user_groups.keys())[int(uid) - 1] if group in user_groups[uname]: return HTTPResponse(request, 200) else: return HTTPResponse(request, 404)
def revoke_token_request_handler(request): assert request.method == 'POST', request.method auth_header = request.headers.get('Authorization') if auth_header: resp = BytesIO(json.dumps({'active': False}).encode('utf8')) return HTTPResponse(request=request, code=200, buffer=resp) else: return HTTPResponse(request=request, code=401)
def forecast_mock(file): body = file_from_resource(f"{file}.json") request = HTTPRequest(method='POST', body=json.dumps(body), url='about:blank') resp = HTTPResponse(request, HTTPStatus.OK, buffer=json.dumps({})) resp._body = json.dumps(body) resp.text = json.dumps(body) resp.status_code = HTTPStatus.OK return resp
def access_token(request): """Handler for access token endpoint Checks code and allocates a new token. Replies with JSON model for the token. """ assert request.method == 'POST', request.method if token_request_style == 'json': body = request.body.decode('utf8') try: body = json.loads(body) except ValueError: return HTTPResponse( request=request, code=400, reason="Body not JSON: %r" % body, ) else: code = body['code'] else: query = urlparse(request.url).query if not query: query = request.body.decode('utf8') query = parse_qs(query) if 'code' not in query: return HTTPResponse( request=request, code=400, reason="No code in access token request: url=%s, body=%s" % (request.url, request.body)) code = query['code'][0] if code not in oauth_codes: return HTTPResponse( request=request, code=403, reason="No such code: %s" % code, ) # consume code, allocate token dt = datetime.now() + timedelta(hours=1) token = jwt.encode({ 'exp': dt }, 'secret', algorithm='HS256').decode('ascii') user = oauth_codes.pop(code) access_tokens[token] = user model = { 'access_token': token, 'refresh_token': token, 'token_type': token_type, } if token_request_style == 'jwt': model['id_token'] = user['id_token'] return model
def test_load(self, mock_httpclient_fetch): cache = stub(get=lambda url: None, add=lambda url, content: None) response = HTTPResponse(HTTPRequest('http://tests.python-zeep.org/test.xml'), 200) response.buffer = True response._body = 'x' mock_httpclient_fetch.return_value = response transport = TornadoAsyncTransport(cache=cache) result = transport.load('http://tests.python-zeep.org/test.xml') assert result == 'x'
def test_tornado_load(mock_httpclient_fetch): cache = stub(get=lambda url: None, add=lambda url, content: None) response = HTTPResponse(HTTPRequest("http://tests.python-zeep.org/test.xml"), 200) response.buffer = True response._body = "x" mock_httpclient_fetch.return_value = response transport = TornadoAsyncTransport(cache=cache) result = transport.load("http://tests.python-zeep.org/test.xml") assert result == "x"
def build_response(self, request, response): resp = HTTPResponse(request, response.status, headers=response.headers, effective_url=request.url, error=None, buffer="") resp._body = response.data f = Future() f.content = None if response.status < 200 or response.status >= 300: resp.error = HTTPError(response.status, response=resp) ioloop.IOLoop().current().add_callback(f.set_exception, resp.error) else: ioloop.IOLoop().current().add_callback(f.set_result, resp) return f
def fetch_impl(self, request, callback): expected = 'http://swifthost/swift/v1/api/path/' \ + 'wikipedia-en-local-public/d/d3/1Mcolors.png' if request.request.url == expected: path = os.path.join(os.path.dirname(__file__), 'originals', '1Mcolors.png') with open(path, 'rb') as f: body = f.read() callback(HTTPResponse(request, 200, buffer=body)) else: callback(HTTPResponse(request, 404))
def org_membership(request): urlinfo = urlparse(request.url) urlmatch = org_membership_regex.match(urlinfo.path) org = urlmatch.group(1) username = urlmatch.group(2) print('Request org = %s, username = %s' % (org, username)) if org not in orgs: print('Org not found: org = %s' % (org)) return HTTPResponse(request, 404) if username not in orgs[org]: print('Member not found: org = %s, username = %s' % (org, username)) return HTTPResponse(request, 404) return HTTPResponse(request, 204)
def access_token(request): """Handler for access token endpoint Checks code and allocates a new token. Replies with JSON model for the token. """ assert request.method == 'POST' if token_request_style == 'json': body = request.body.decode('utf8') try: body = json.loads(body) except ValueError: return HTTPResponse( request=request, code=400, reason="Body not JSON: %r" % body, ) else: code = body['code'] else: query = urlparse(request.url).query if not query: query = request.body.decode('utf8') query = parse_qs(query) if 'code' not in query: return HTTPResponse( request=request, code=400, reason="No code in access token request: url=%s, body=%s" % ( request.url, request.body, )) code = query['code'][0] if code not in oauth_codes: app_log.warning() return HTTPResponse( request=request, code=403, reason="No such code: %s" % code, ) # consume code, allocate token token = uuid.uuid4().hex user = oauth_codes.pop(code) access_tokens[token] = user return { 'access_token': token, 'token_type': token_type, }
def team_membership(request): urlinfo = urlparse(request.url) urlmatch = membership_regex.match(urlinfo.path) team = urlmatch.group(1) username = urlmatch.group(2) print('Request team = %s, username = %s' % (team, username)) if team not in teams: print('Team not found: team = %s' % (team)) return HTTPResponse(request, 404) if username not in teams[team]: print('Member not found: team = %s, username = %s' % (team, username)) return HTTPResponse(request, 404) return HTTPResponse(request, 204)
def is_member(request): urlinfo = urlparse(request.url) project_id, uid = member_regex.match(urlinfo.path).group(1, 2) if user_projects.get(project_id) and user_projects.get(project_id).get(uid): res = user_projects.get(project_id).get(uid) return HTTPResponse(request=request, code=200, buffer=BytesIO(json.dumps(res).encode('utf8')), headers={'Content-Type': 'application/json'}, ) else: return HTTPResponse(request=request, code=404, buffer=BytesIO(''.encode('utf8')) )
async def _make_http_response( handler: RequestHandler, code: int = 200, reason: str = "OK", headers: HTTPHeaders = HTTPHeaders({"content-type": "application/json"}), effective_url: str = "http://hub.example.com/", body: Dict[str, str] = {"foo": "bar"}, ) -> HTTPResponse: """ Creates an HTTPResponse object from a given request. Args: handler: tornado.web.RequestHandler object. code: response code, e.g. 200 or 404 reason: reason phrase describing the status code headers: HTTPHeaders (response header object), use the dict within the constructor, e.g. {"content-type": "application/json"} effective_url: final location of the resource after following any redirects body: dictionary that represents the StringIO (buffer) body Returns: A tornado.client.HTTPResponse object """ dict_to_buffer = StringIO(json.dumps(body)) if body is not None else None return HTTPResponse( request=handler, code=code, reason=reason, headers=headers, effective_url=effective_url, buffer=dict_to_buffer, )
def _on_timeout(self): self._timeout = None if self.callback is not None: self.callback(HTTPResponse(self.request, 599, error=HTTPError(599, "Timeout"))) self.callback = None self.stream.close()
def _on_body(self, data): if self._timeout is not None: self.io_loop.remove_timeout(self._timeout) self._timeout = None original_request = getattr(self.request, "original_request", self.request) if (self.request.follow_redirects and self.request.max_redirects > 0 and self.code in (301, 302, 303, 307)): assert isinstance(self.request, _RequestProxy) new_request = copy.copy(self.request.request) new_request.url = urlparse.urljoin(self.request.url, self.headers["Location"]) new_request.max_redirects = self.request.max_redirects - 1 del new_request.headers["Host"] # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 # Client SHOULD make a GET request after a 303. # According to the spec, 302 should be followed by the same # method as the original request, but in practice browsers # treat 302 the same as 303, and many servers use 302 for # compatibility with pre-HTTP/1.1 user agents which don't # understand the 303 status. if self.code in (302, 303): new_request.method = "GET" new_request.body = None for h in [ "Content-Length", "Content-Type", "Content-Encoding", "Transfer-Encoding" ]: try: del self.request.headers[h] except KeyError: pass new_request.original_request = original_request final_callback = self.final_callback self.final_callback = None self._release() self.client.fetch(new_request, final_callback) self.stream.close() return if self._decompressor: data = (self._decompressor.decompress(data) + self._decompressor.flush()) if self.request.streaming_callback: if self.chunks is None: # if chunks is not None, we already called streaming_callback # in _on_chunk_data self.request.streaming_callback(data) buffer = BytesIO() else: buffer = BytesIO(data) # TODO: don't require one big string? response = HTTPResponse(original_request, self.code, reason=self.reason, headers=self.headers, request_time=self.io_loop.time() - self.start_time, buffer=buffer, effective_url=self.request.url) self._run_callback(response) self.stream.close()
def response_from_debug(request, response): debug_response = etree.XML(response.body) original_response = debug_response.find('original-response') if original_response: response_info = frontik.xml_util.xml_to_dict(original_response) original_response.getparent().remove(original_response) original_buffer = base64.b64decode(response_info['buffer']) headers = dict(response.headers) if response_info['headers']: headers.update(response_info['headers']) fake_response = HTTPResponse(request, int(response_info['code']), headers=HTTPHeaders(headers), buffer=BytesIO(original_buffer), effective_url=response.effective_url, request_time=response.request_time, time_info=response.time_info) return debug_response, fake_response return None
def wechat_api_mock(client, request, *args, **kwargs): url = urlparse(request.url) path = url.path.replace('/cgi-bin/', '').replace('/', '_') if path.startswith('_'): path = path[1:] res_file = os.path.join(_FIXTURE_PATH, '%s.json' % path) content = { 'errcode': 99999, 'errmsg': 'can not find fixture %s' % res_file, } headers = { 'Content-Type': 'application/json' } try: with open(res_file, 'rb') as f: content = f.read().decode('utf-8') except (IOError, ValueError) as e: content['errmsg'] = 'Loads fixture {0} failed, error: {1}'.format( res_file, e ) content = json.dumps(content) buffer = StringIO(content) resp = HTTPResponse( request, 200, headers=headers, buffer=buffer, ) future = Future() future.set_result(resp) return future
def test_downstream_success_blacklisted_headers_removed( self, mock_awsproxy): # Given mock_execute_downstream = CoroutineMock() mock_execute_downstream.return_value = HTTPResponse( request=HTTPRequest(url="https://awsservice.amazonaws.com/"), code=200, headers=HTTPHeaders({ "Host": "awsservice.amazonaws.com", "X-Amz-RequestId": "foo-abc", "Transfer-Encoding": "chunked", }), buffer=BytesIO(b"SomeResponse"), ) mock_instance = mock_awsproxy.return_value mock_instance.execute_downstream = mock_execute_downstream # When response = self.fetch("/awsproxy") # Then mock_execute_downstream.assert_awaited_once() assert 200 == response.code assert b"SomeResponse" == response.body assert "Transfer-Encoding" not in response.headers assert "foo-abc" == response.headers["X-Amz-RequestId"] assert "awsservice.amazonaws.com" == response.headers["Host"]
def create(self, param, req): tableName = param['tableName'].name initVal = param['initVal'] if tableName in self.tables: raise KeyError, 'Table name already existed' #print 'MASTER CREATE:', initVal self.tables[tableName] = { 'len': 0, } #Data Partition vals = {} for worker_id in xrange(self.num_workers): vals[worker_id] = {} for key in initVal: #Bookkeeping self.tables[tableName]['len'] += 1 self.tables[tableName][key] = type(initVal[key]) worker = hash(key) % self.num_workers vals[worker][key] = initVal[key] futures = [] for worker_id in xrange(self.num_workers): args = {'tableName': tableName, 'initVal': vals[worker_id]} futures.append( self.client.fetch( formatQuery(self.workers[worker_id], 'create', args))) fu = Future() req.url = self.host fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK'))) return fu
def append(self, param, req): tableName = param['tableName'].name val = param['val'] if not tableName in self.tables: raise KeyError, 'Table Not Found!' args = { 'tableName': tableName, 'key': self.tables[tableName]['len'], 'val': val } worker = self.workers[hash(self.tables[tableName]['len']) % self.num_workers] #print 'MASTER -> {0} Append: {1} to key {2}'.format(worker, val, self.tables[tableName]['len']) self.client.fetch(formatQuery(worker, 'set', args)) fu = Future() req.url = self.host fu.set_result( HTTPResponse(req, 200, buffer=cStringIO.StringIO( str(self.tables[tableName]['len'])))) self.tables[tableName]['len'] += 1 return fu
def _process_queue(self): with stack_context.NullContext(): while True: started = 0 while self._free_list and self._requests: started += 1 curl = self._free_list.pop() (request, callback) = self._requests.popleft() curl.info = { "headers": httputil.HTTPHeaders(), "buffer": BytesIO(), "request": request, "callback": callback, "curl_start_time": time.time(), } try: self._curl_setup_request(curl, request, curl.info["buffer"], curl.info["headers"]) except Exception as e: # If there was an error in setup, pass it on # to the callback. Note that allowing the # error to escape here will appear to work # most of the time since we are still in the # caller's original stack frame, but when # _process_queue() is called from # _finish_pending_requests the exceptions have # nowhere to go. callback( HTTPResponse(request=request, code=599, error=e)) else: self._multi.add_handle(curl) if not started: break
def _response(self): request = HTTPRequest("") status_code = self.fixture.get("code", 200) headers = self.fixture.get("headers", {}) body = StringIO(self.fixture.get("body", "")) return HTTPResponse(request, status_code, headers, body)
def response_from_debug(request, response): debug_response = etree.XML(response.body) original_response = debug_response.find('original-response') if original_response is not None: response_info = xml_to_dict(original_response) original_response.getparent().remove(original_response) original_buffer = base64.b64decode(response_info.get('buffer', '')) headers = dict(response.headers) response_info_headers = response_info.get('headers', {}) if response_info_headers: headers.update(response_info_headers) fake_response = HTTPResponse( request, int(response_info.get('code', 599)), headers=HTTPHeaders(headers), buffer=BytesIO(original_buffer), effective_url=response.effective_url, request_time=response.request_time, time_info=response.time_info ) return debug_response, fake_response return None
def _handle_exception(self, typ, value, tb): if self.final_callback: self._remove_timeout() if isinstance(value, StreamClosedError): value = HTTPError(599, "Stream closed") self._run_callback( HTTPResponse( self.request, 599, error=value, request_time=self.io_loop.time() - self.start_time, )) if hasattr(self, "stream"): # TODO: this may cause a StreamClosedError to be raised # by the connection's Future. Should we cancel the # connection more gracefully? self.stream.close() return True else: # If our callback has already been called, we are probably # catching an exception that is not caused by us but rather # some child of our callback. Rather than drop it on the floor, # pass it along, unless it's just the stream being closed. return isinstance(value, StreamClosedError)
def finish(self) -> None: assert self.code is not None data = b"".join(self.chunks) self._remove_timeout() original_request = getattr(self.request, "original_request", self.request) if self._should_follow_redirect(): assert isinstance(self.request, _RequestProxy) new_request = copy.copy(self.request.request) new_request.url = urllib.parse.urljoin(self.request.url, self.headers["Location"]) new_request.max_redirects = self.request.max_redirects - 1 del new_request.headers["Host"] # https://tools.ietf.org/html/rfc7231#section-6.4 # # The original HTTP spec said that after a 301 or 302 # redirect, the request method should be preserved. # However, browsers implemented this by changing the # method to GET, and the behavior stuck. 303 redirects # always specified this POST-to-GET behavior (arguably 303 # redirects should change *all* requests to GET, but # libcurl only does this for POST so we follow their # example). if self.code in (301, 302, 303) and self.request.method == "POST": new_request.method = "GET" new_request.body = None for h in [ "Content-Length", "Content-Type", "Content-Encoding", "Transfer-Encoding", ]: try: del self.request.headers[h] except KeyError: pass new_request.original_request = original_request final_callback = self.final_callback self.final_callback = None self._release() fut = self.client.fetch(new_request, raise_error=False) fut.add_done_callback(lambda f: final_callback(f.result())) self._on_end_request() return if self.request.streaming_callback: buffer = BytesIO() else: buffer = BytesIO(data) # TODO: don't require one big string? response = HTTPResponse( original_request, self.code, reason=getattr(self, "reason", None), headers=self.headers, request_time=self.io_loop.time() - self.start_time, start_time=self.start_wall_time, buffer=buffer, effective_url=self.request.url, ) self._run_callback(response) self._on_end_request()
def get_groups_request_handler(request): mock_globus_groups_response = [ { 'id': '21c6bc5d-fc12-4f60-b999-76766cd596c2', 'my_memberships': [{ 'role': 'manager' }], }, { 'id': '915dcd61-c842-4ea4-97c6-57396b936016', 'my_memberships': [{ 'role': 'member' }], }, { 'id': 'd11abe71-5132-4c04-a4ad-50926885dc8c', 'my_memberships': [{ 'role': 'member', }], }, ] assert request.method == 'GET', request.method resp = BytesIO(json.dumps(mock_globus_groups_response).encode('utf-8')) return HTTPResponse( request=request, code=200, headers={'Content-Type': 'application/json'}, buffer=resp, )
async def test_clear_subscriptions_synapse(patch, app, async_mock, status_code): app.app_id = 'my_cool_app' app.config.ASYNCY_SYNAPSE_HOST = 'syn' app.config.ASYNCY_SYNAPSE_PORT = 9000 expected_url = f'http://{app.config.ASYNCY_SYNAPSE_HOST}:' \ f'{app.config.ASYNCY_SYNAPSE_PORT}/clear_all' expected_kwargs = { 'method': 'POST', 'body': json.dumps({'app_id': app.app_id}), 'headers': { 'Content-Type': 'application/json; charset=utf-8' } } res = HTTPResponse(HTTPRequest(url=expected_url), status_code) patch.object(HttpUtils, 'fetch_with_retry', new=async_mock(return_value=res)) ret = await app.clear_subscriptions_synapse() HttpUtils.fetch_with_retry.mock.assert_called_with(3, app.logger, expected_url, AsyncHTTPClient(), expected_kwargs) if status_code == 200: assert ret is True else: assert ret is False
def _on_body(self, data): if self._timeout is not None: self.io_loop.remove_timeout(self._timeout) self._timeout = None if self._decompressor: data = self._decompressor.decompress(data) if self.request.streaming_callback: if self.chunks is None: # if chunks is not None, we already called streaming_callback # in _on_chunk_data self.request.streaming_callback(data) buffer = StringIO() else: buffer = StringIO(data) # TODO: don't require one big string? original_request = getattr(self.request, "original_request", self.request) if (self.request.follow_redirects and self.request.max_redirects > 0 and self.code in (301, 302)): new_request = copy.copy(self.request) new_request.url = urlparse.urljoin(self.request.url, self.headers["Location"]) new_request.max_redirects -= 1 del new_request.headers["Host"] new_request.original_request = original_request self.client.fetch(new_request, self.callback) self.callback = None return response = HTTPResponse(original_request, self.code, headers=self.headers, buffer=buffer, effective_url=self.request.url) self.callback(response) self.callback = None
def handle_http_connection(self, conn): # Read the trailing HTTP request and process it with protocol_switcher. # We can't rely on ioloop to trigger read because it has been already # triggered for SSL handshake. payload = yield conn.stream.read_bytes(1024, partial=True) logger.debug("Received %r", payload[:128]) # Simulate conn._read_message side effect. This is required by # HTTP1Connection.write_headers() conn._request_start_line = parse_request_start_line('GET / HTTP/1.1') try: start_line, headers = parse_http_headers(payload) conn._request_start_line = start_line request = HTTPRequest( connection=conn, headers=headers, start_line=start_line, ) request.config = self.request_callback.config response = protocol_switcher(request) except Exception as e: logger.error("Failed to switch to HTTPS: %s", e) response = HTTPResponse( request=object(), code=500, headers=HTTPHeaders({'Content-Length': '0'}), effective_url='https://useless_effective_url') yield conn.write_headers( start_line=ResponseStartLine( 'HTTP/1.1', response.code, response.reason, ), headers=response.headers, )
def test_str(self): response = HTTPResponse( # type: ignore HTTPRequest("http://example.com"), 200, buffer=BytesIO() ) s = str(response) self.assertTrue(s.startswith("HTTPResponse(")) self.assertIn("code=200", s)
def test_post(self, mock_httpclient_fetch): cache = stub(get=lambda url: None, add=lambda url, content: None) response = HTTPResponse(HTTPRequest('http://tests.python-zeep.org/test.xml'), 200) response.buffer = True response._body = 'x' http_fetch_future = Future() http_fetch_future.set_result(response) mock_httpclient_fetch.return_value = http_fetch_future transport = TornadoAsyncTransport(cache=cache) envelope = etree.Element('Envelope') result = yield transport.post_xml( 'http://tests.python-zeep.org/test.xml', envelope=envelope, headers={}) assert result.content == 'x' assert result.status_code == 200
def fetch_impl(self, request, response_callback): urlinfo = urlparse(request.url) host = urlinfo.hostname if host not in self.hosts: app_log.warning("Not mocking request to %s", request.url) return super().fetch_impl(request, response_callback) paths = self.hosts[host] response = None for path_spec, handler in paths: if isinstance(path_spec, str): if path_spec == urlinfo.path: response = handler(request) break else: if path_spec.match(urlinfo.path): response = handler(request) break if response is None: response = HTTPResponse(request=request, code=404, reason=request.url) elif isinstance(response, int): response = HTTPResponse(request=request, code=response) elif isinstance(response, bytes): response = HTTPResponse(request=request, code=200, buffer=BytesIO(response), ) elif isinstance(response, str): response = HTTPResponse(request=request, code=200, buffer=BytesIO(response.encode('utf8')), ) elif isinstance(response, (dict, list)): response = HTTPResponse(request=request, code=200, buffer=BytesIO(json.dumps(response).encode('utf8')), headers={'Content-Type': 'application/json'}, ) response_callback(response)
def fetch(self, url, method='GET', body=None, headers=None, fail=True, freeze=False, follow_redirects=True, max_redirects=5, **kwargs): if not headers: headers = {} default_headers = copy(self._headers) default_headers.update(headers) headers = default_headers if body is not None and 'Content-Type' not in headers: headers['Content-Type'] = 'application/json' if method in self.METHODS_WITH_BODY and 'body_producer' not in kwargs: body = body or '' if headers.get('Content-Type', '') == 'application/json': body = self._make_json(body) params = copy(self._default_args) params.update(kwargs) last_exc = RuntimeError("Something wrong") for _ in range(max_redirects + 1): request = HTTPRequest( b(url), method=method, body=body, headers=HTTPHeaders(headers), follow_redirects=False, **params ) request.headers['Cookie'] = '; '.join( '{0.key}={0.value}'.format(cookie) for cookie in self._cookies.values() ) need_redirect = False try: response = yield self._client.fetch(request) response.fail = False except HTTPError as e: last_exc = e response = e.response if e.code == 599: response = HTTPResponse( request=request, code=e.code, headers=HTTPHeaders(), effective_url=request.url ) response.fail = True if e.code in (301, 302, 303, 307) and follow_redirects: need_redirect = True else: response.fail = True if fail and e.response: content_type = e.response.headers.get('Content-Type', '') e.response._body = self._decode_body(content_type, response.body) if e.response.body and 'application/json' in content_type.lower(): e.response._body = self._parse_json(e.response.body) if not need_redirect: break if not freeze: for cookie in response.headers.get_list('Set-Cookie'): self._cookies.load(cookie) else: response.fail = True if fail and response.fail: raise last_exc content_type = response.headers.get('Content-Type', '') response._body = self._decode_body(content_type, response.body) if response.body and 'json' in response.headers.get('Content-Type', ''): new_body = self._parse_json(response.body) response._body = _freeze_response(new_body) if not freeze: for cookie in response.headers.get_list('Set-Cookie'): self._cookies.load(cookie) raise Return(response)