def revoke_token_request_handler(request):
assert request.method == 'POST', request.method
auth_header = request.headers.get('Authorization')
if auth_header:
resp = BytesIO(json.dumps({'active': False}).encode('utf8'))
return HTTPResponse(request=request, code=200, buffer=resp)
else:
return HTTPResponse(request=request, code=401)
def is_member(request):
urlinfo = urlparse(request.url)
group, uid = member_regex.match(urlinfo.path).group(1, 2)
uname = list(user_groups.keys())[int(uid) - 1]
if group in user_groups[uname]:
return HTTPResponse(request, 200)
else:
return HTTPResponse(request, 404)
def access_token(request):
"""Handler for access token endpoint
Checks code and allocates a new token.
Replies with JSON model for the token.
"""
assert request.method == 'POST', request.method
if token_request_style == 'json':
body = request.body.decode('utf8')
try:
body = json.loads(body)
except ValueError:
return HTTPResponse(
request=request,
code=400,
reason="Body not JSON: %r" % body,
)
else:
code = body['code']
else:
query = urlparse(request.url).query
if not query:
query = request.body.decode('utf8')
query = parse_qs(query)
if 'code' not in query:
return HTTPResponse(
request=request,
code=400,
reason="No code in access token request: url=%s, body=%s" %
(request.url, request.body))
code = query['code'][0]
if code not in oauth_codes:
return HTTPResponse(
request=request,
code=403,
reason="No such code: %s" % code,
)
# consume code, allocate token
dt = datetime.now() + timedelta(hours=1)
token = jwt.encode({
'exp': dt
}, 'secret', algorithm='HS256').decode('ascii')
user = oauth_codes.pop(code)
access_tokens[token] = user
model = {
'access_token': token,
'refresh_token': token,
'token_type': token_type,
}
if token_request_style == 'jwt':
model['id_token'] = user['id_token']
return model
def fetch_impl(self, request, callback):
expected = 'http://swifthost/swift/v1/api/path/' \
+ 'wikipedia-en-local-public/d/d3/1Mcolors.png'
if request.request.url == expected:
path = os.path.join(os.path.dirname(__file__), 'originals',
'1Mcolors.png')
with open(path, 'rb') as f:
body = f.read()
callback(HTTPResponse(request, 200, buffer=body))
else:
callback(HTTPResponse(request, 404))
def org_membership(request):
urlinfo = urlparse(request.url)
urlmatch = org_membership_regex.match(urlinfo.path)
org = urlmatch.group(1)
username = urlmatch.group(2)
print('Request org = %s, username = %s' % (org, username))
if org not in orgs:
print('Org not found: org = %s' % (org))
return HTTPResponse(request, 404)
if username not in orgs[org]:
print('Member not found: org = %s, username = %s' % (org, username))
return HTTPResponse(request, 404)
return HTTPResponse(request, 204)
def team_membership(request):
urlinfo = urlparse(request.url)
urlmatch = membership_regex.match(urlinfo.path)
team = urlmatch.group(1)
username = urlmatch.group(2)
print('Request team = %s, username = %s' % (team, username))
if team not in teams:
print('Team not found: team = %s' % (team))
return HTTPResponse(request, 404)
if username not in teams[team]:
print('Member not found: team = %s, username = %s' %
(team, username))
return HTTPResponse(request, 404)
return HTTPResponse(request, 204)
def is_member(request):
urlinfo = urlparse(request.url)
project_id, uid = member_regex.match(urlinfo.path).group(1, 2)
if user_projects.get(project_id) and user_projects.get(project_id).get(uid):
res = user_projects.get(project_id).get(uid)
return HTTPResponse(request=request, code=200,
buffer=BytesIO(json.dumps(res).encode('utf8')),
headers={'Content-Type': 'application/json'},
)
else:
return HTTPResponse(request=request, code=404,
buffer=BytesIO(''.encode('utf8'))
)
def access_token(request):
"""Handler for access token endpoint
Checks code and allocates a new token.
Replies with JSON model for the token.
"""
assert request.method == 'POST'
if token_request_style == 'json':
body = request.body.decode('utf8')
try:
body = json.loads(body)
except ValueError:
return HTTPResponse(
request=request,
code=400,
reason="Body not JSON: %r" % body,
)
else:
code = body['code']
else:
query = urlparse(request.url).query
if not query:
query = request.body.decode('utf8')
query = parse_qs(query)
if 'code' not in query:
return HTTPResponse(
request=request,
code=400,
reason="No code in access token request: url=%s, body=%s" %
(
request.url,
request.body,
))
code = query['code'][0]
if code not in oauth_codes:
app_log.warning()
return HTTPResponse(
request=request,
code=403,
reason="No such code: %s" % code,
)
# consume code, allocate token
token = uuid.uuid4().hex
user = oauth_codes.pop(code)
access_tokens[token] = user
return {
'access_token': token,
'token_type': token_type,
}
def _on_body(self, data):
if self._timeout is not None:
self.io_loop.remove_timeout(self._timeout)
self._timeout = None
if self._decompressor:
data = self._decompressor.decompress(data)
if self.request.streaming_callback:
if self.chunks is None:
# if chunks is not None, we already called streaming_callback
# in _on_chunk_data
self.request.streaming_callback(data)
buffer = StringIO()
else:
buffer = StringIO(data) # TODO: don't require one big string?
original_request = getattr(self.request, "original_request",
self.request)
if (self.request.follow_redirects and self.request.max_redirects > 0
and self.code in (301, 302)):
new_request = copy.copy(self.request)
new_request.url = urlparse.urljoin(self.request.url,
self.headers["Location"])
new_request.max_redirects -= 1
del new_request.headers["Host"]
new_request.original_request = original_request
self.client.fetch(new_request, self.callback)
self.callback = None
return
response = HTTPResponse(original_request,
self.code,
headers=self.headers,
buffer=buffer,
effective_url=self.request.url)
self.callback(response)
self.callback = None
def handle_http_connection(self, conn):
# Read the trailing HTTP request and process it with protocol_switcher.
# We can't rely on ioloop to trigger read because it has been already
# triggered for SSL handshake.
payload = yield conn.stream.read_bytes(1024, partial=True)
logger.debug("Received %r", payload[:128])
# Simulate conn._read_message side effect. This is required by
# HTTP1Connection.write_headers()
conn._request_start_line = parse_request_start_line('GET / HTTP/1.1')
try:
start_line, headers = parse_http_headers(payload)
conn._request_start_line = start_line
request = HTTPRequest(
connection=conn,
headers=headers,
start_line=start_line,
)
request.config = self.request_callback.config
response = protocol_switcher(request)
except Exception as e:
logger.error("Failed to switch to HTTPS: %s", e)
response = HTTPResponse(
request=object(),
code=500,
headers=HTTPHeaders({'Content-Length': '0'}),
effective_url='https://useless_effective_url')
yield conn.write_headers(
start_line=ResponseStartLine(
'HTTP/1.1',
response.code,
response.reason,
),
headers=response.headers,
)
def test_str(self):
response = HTTPResponse( # type: ignore
HTTPRequest("http://example.com"), 200, buffer=BytesIO()
)
s = str(response)
self.assertTrue(s.startswith("HTTPResponse("))
self.assertIn("code=200", s)
def wechat_api_mock(client, request, *args, **kwargs):
url = urlparse(request.url)
path = url.path.replace('/cgi-bin/', '').replace('/', '_')
if path.startswith('_'):
path = path[1:]
res_file = os.path.join(_FIXTURE_PATH, '%s.json' % path)
content = {
'errcode': 99999,
'errmsg': 'can not find fixture %s' % res_file,
}
headers = {
'Content-Type': 'application/json'
}
try:
with open(res_file, 'rb') as f:
content = f.read().decode('utf-8')
except (IOError, ValueError) as e:
content['errmsg'] = 'Loads fixture {0} failed, error: {1}'.format(
res_file,
e
)
content = json.dumps(content)
buffer = StringIO(content)
resp = HTTPResponse(
request,
200,
headers=headers,
buffer=buffer,
)
future = Future()
future.set_result(resp)
return future
def _on_body(self, data):
if self._timeout is not None:
self.io_loop.remove_timeout(self._timeout)
self._timeout = None
original_request = getattr(self.request, "original_request",
self.request)
if (self.request.follow_redirects and self.request.max_redirects > 0
and self.code in (301, 302, 303, 307)):
assert isinstance(self.request, _RequestProxy)
new_request = copy.copy(self.request.request)
new_request.url = urlparse.urljoin(self.request.url,
self.headers["Location"])
new_request.max_redirects = self.request.max_redirects - 1
del new_request.headers["Host"]
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
# Client SHOULD make a GET request after a 303.
# According to the spec, 302 should be followed by the same
# method as the original request, but in practice browsers
# treat 302 the same as 303, and many servers use 302 for
# compatibility with pre-HTTP/1.1 user agents which don't
# understand the 303 status.
if self.code in (302, 303):
new_request.method = "GET"
new_request.body = None
for h in [
"Content-Length", "Content-Type", "Content-Encoding",
"Transfer-Encoding"
]:
try:
del self.request.headers[h]
except KeyError:
pass
new_request.original_request = original_request
final_callback = self.final_callback
self.final_callback = None
self._release()
self.client.fetch(new_request, final_callback)
self.stream.close()
return
if self._decompressor:
data = (self._decompressor.decompress(data) +
self._decompressor.flush())
if self.request.streaming_callback:
if self.chunks is None:
# if chunks is not None, we already called streaming_callback
# in _on_chunk_data
self.request.streaming_callback(data)
buffer = BytesIO()
else:
buffer = BytesIO(data) # TODO: don't require one big string?
response = HTTPResponse(original_request,
self.code,
reason=self.reason,
headers=self.headers,
request_time=self.io_loop.time() -
self.start_time,
buffer=buffer,
effective_url=self.request.url)
self._run_callback(response)
self.stream.close()
def response_from_debug(request, response):
debug_response = etree.XML(response.body)
original_response = debug_response.find('original-response')
if original_response is not None:
response_info = xml_to_dict(original_response)
original_response.getparent().remove(original_response)
original_buffer = base64.b64decode(response_info.get('buffer', ''))
headers = dict(response.headers)
response_info_headers = response_info.get('headers', {})
if response_info_headers:
headers.update(response_info_headers)
fake_response = HTTPResponse(
request,
int(response_info.get('code', 599)),
headers=HTTPHeaders(headers),
buffer=BytesIO(original_buffer),
effective_url=response.effective_url,
request_time=response.request_time,
time_info=response.time_info
)
return debug_response, fake_response
return None
def _on_timeout(self):
self._timeout = None
if self.callback is not None:
self.callback(HTTPResponse(self.request, 599,
error=HTTPError(599, "Timeout")))
self.callback = None
self.stream.close()
def test_error_with_response(self):
resp = HTTPResponse(HTTPRequest('http://example.com/'), 403)
with self.assertRaises(HTTPError) as cm:
resp.rethrow()
e = cm.exception
self.assertEqual(str(e), "HTTP 403: Forbidden")
self.assertEqual(repr(e), "HTTP 403: Forbidden")
def create(self, param, req):
tableName = param['tableName'].name
initVal = param['initVal']
if tableName in self.tables:
raise KeyError, 'Table name already existed'
#print 'MASTER CREATE:', initVal
self.tables[tableName] = {
'len': 0,
}
#Data Partition
vals = {}
for worker_id in xrange(self.num_workers):
vals[worker_id] = {}
for key in initVal:
#Bookkeeping
self.tables[tableName]['len'] += 1
self.tables[tableName][key] = type(initVal[key])
worker = hash(key) % self.num_workers
vals[worker][key] = initVal[key]
futures = []
for worker_id in xrange(self.num_workers):
args = {'tableName': tableName, 'initVal': vals[worker_id]}
futures.append(
self.client.fetch(
formatQuery(self.workers[worker_id], 'create', args)))
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def append(self, param, req):
tableName = param['tableName'].name
val = param['val']
if not tableName in self.tables:
raise KeyError, 'Table Not Found!'
args = {
'tableName': tableName,
'key': self.tables[tableName]['len'],
'val': val
}
worker = self.workers[hash(self.tables[tableName]['len']) %
self.num_workers]
#print 'MASTER -> {0} Append: {1} to key {2}'.format(worker, val, self.tables[tableName]['len'])
self.client.fetch(formatQuery(worker, 'set', args))
fu = Future()
req.url = self.host
fu.set_result(
HTTPResponse(req,
200,
buffer=cStringIO.StringIO(
str(self.tables[tableName]['len']))))
self.tables[tableName]['len'] += 1
return fu
def _process_queue(self):
with stack_context.NullContext():
while True:
started = 0
while self._free_list and self._requests:
started += 1
curl = self._free_list.pop()
(request, callback) = self._requests.popleft()
curl.info = {
"headers": httputil.HTTPHeaders(),
"buffer": BytesIO(),
"request": request,
"callback": callback,
"curl_start_time": time.time(),
}
try:
self._curl_setup_request(curl, request,
curl.info["buffer"],
curl.info["headers"])
except Exception as e:
# If there was an error in setup, pass it on
# to the callback. Note that allowing the
# error to escape here will appear to work
# most of the time since we are still in the
# caller's original stack frame, but when
# _process_queue() is called from
# _finish_pending_requests the exceptions have
# nowhere to go.
callback(
HTTPResponse(request=request, code=599, error=e))
else:
self._multi.add_handle(curl)
if not started:
break
def _response(self):
request = HTTPRequest("")
status_code = self.fixture.get("code", 200)
headers = self.fixture.get("headers", {})
body = StringIO(self.fixture.get("body", ""))
return HTTPResponse(request, status_code, headers, body)
async def test_clear_subscriptions_synapse(patch, app, async_mock,
status_code):
app.app_id = 'my_cool_app'
app.config.ASYNCY_SYNAPSE_HOST = 'syn'
app.config.ASYNCY_SYNAPSE_PORT = 9000
expected_url = f'http://{app.config.ASYNCY_SYNAPSE_HOST}:' \
f'{app.config.ASYNCY_SYNAPSE_PORT}/clear_all'
expected_kwargs = {
'method': 'POST',
'body': json.dumps({'app_id': app.app_id}),
'headers': {
'Content-Type': 'application/json; charset=utf-8'
}
}
res = HTTPResponse(HTTPRequest(url=expected_url), status_code)
patch.object(HttpUtils,
'fetch_with_retry',
new=async_mock(return_value=res))
ret = await app.clear_subscriptions_synapse()
HttpUtils.fetch_with_retry.mock.assert_called_with(3, app.logger,
expected_url,
AsyncHTTPClient(),
expected_kwargs)
if status_code == 200:
assert ret is True
else:
assert ret is False
def response_from_debug(request, response):
debug_response = etree.XML(response.body)
original_response = debug_response.find('original-response')
if original_response:
response_info = frontik.xml_util.xml_to_dict(original_response)
original_response.getparent().remove(original_response)
original_buffer = base64.b64decode(response_info['buffer'])
headers = dict(response.headers)
if response_info['headers']:
headers.update(response_info['headers'])
fake_response = HTTPResponse(request,
int(response_info['code']),
headers=HTTPHeaders(headers),
buffer=BytesIO(original_buffer),
effective_url=response.effective_url,
request_time=response.request_time,
time_info=response.time_info)
return debug_response, fake_response
return None
async def _make_http_response(
handler: RequestHandler,
code: int = 200,
reason: str = "OK",
headers: HTTPHeaders = HTTPHeaders({"content-type": "application/json"}),
effective_url: str = "http://hub.example.com/",
body: Dict[str, str] = {"foo": "bar"},
) -> HTTPResponse:
"""
Creates an HTTPResponse object from a given request.
Args:
handler: tornado.web.RequestHandler object.
code: response code, e.g. 200 or 404
reason: reason phrase describing the status code
headers: HTTPHeaders (response header object), use the dict within the constructor, e.g.
{"content-type": "application/json"}
effective_url: final location of the resource after following any redirects
body: dictionary that represents the StringIO (buffer) body
Returns:
A tornado.client.HTTPResponse object
"""
dict_to_buffer = StringIO(json.dumps(body)) if body is not None else None
return HTTPResponse(
request=handler,
code=code,
reason=reason,
headers=headers,
effective_url=effective_url,
buffer=dict_to_buffer,
)
def finish(self) -> None:
assert self.code is not None
data = b"".join(self.chunks)
self._remove_timeout()
original_request = getattr(self.request, "original_request",
self.request)
if self._should_follow_redirect():
assert isinstance(self.request, _RequestProxy)
new_request = copy.copy(self.request.request)
new_request.url = urllib.parse.urljoin(self.request.url,
self.headers["Location"])
new_request.max_redirects = self.request.max_redirects - 1
del new_request.headers["Host"]
# https://tools.ietf.org/html/rfc7231#section-6.4
#
# The original HTTP spec said that after a 301 or 302
# redirect, the request method should be preserved.
# However, browsers implemented this by changing the
# method to GET, and the behavior stuck. 303 redirects
# always specified this POST-to-GET behavior (arguably 303
# redirects should change *all* requests to GET, but
# libcurl only does this for POST so we follow their
# example).
if self.code in (301, 302, 303) and self.request.method == "POST":
new_request.method = "GET"
new_request.body = None
for h in [
"Content-Length",
"Content-Type",
"Content-Encoding",
"Transfer-Encoding",
]:
try:
del self.request.headers[h]
except KeyError:
pass
new_request.original_request = original_request
final_callback = self.final_callback
self.final_callback = None
self._release()
fut = self.client.fetch(new_request, raise_error=False)
fut.add_done_callback(lambda f: final_callback(f.result()))
self._on_end_request()
return
if self.request.streaming_callback:
buffer = BytesIO()
else:
buffer = BytesIO(data) # TODO: don't require one big string?
response = HTTPResponse(
original_request,
self.code,
reason=getattr(self, "reason", None),
headers=self.headers,
request_time=self.io_loop.time() - self.start_time,
start_time=self.start_wall_time,
buffer=buffer,
effective_url=self.request.url,
)
self._run_callback(response)
self._on_end_request()
def test_downstream_success_blacklisted_headers_removed(
self, mock_awsproxy):
# Given
mock_execute_downstream = CoroutineMock()
mock_execute_downstream.return_value = HTTPResponse(
request=HTTPRequest(url="https://awsservice.amazonaws.com/"),
code=200,
headers=HTTPHeaders({
"Host": "awsservice.amazonaws.com",
"X-Amz-RequestId": "foo-abc",
"Transfer-Encoding": "chunked",
}),
buffer=BytesIO(b"SomeResponse"),
)
mock_instance = mock_awsproxy.return_value
mock_instance.execute_downstream = mock_execute_downstream
# When
response = self.fetch("/awsproxy")
# Then
mock_execute_downstream.assert_awaited_once()
assert 200 == response.code
assert b"SomeResponse" == response.body
assert "Transfer-Encoding" not in response.headers
assert "foo-abc" == response.headers["X-Amz-RequestId"]
assert "awsservice.amazonaws.com" == response.headers["Host"]
def _handle_exception(self, typ, value, tb):
if self.final_callback:
self._remove_timeout()
if isinstance(value, StreamClosedError):
value = HTTPError(599, "Stream closed")
self._run_callback(
HTTPResponse(
self.request,
599,
error=value,
request_time=self.io_loop.time() - self.start_time,
))
if hasattr(self, "stream"):
# TODO: this may cause a StreamClosedError to be raised
# by the connection's Future. Should we cancel the
# connection more gracefully?
self.stream.close()
return True
else:
# If our callback has already been called, we are probably
# catching an exception that is not caused by us but rather
# some child of our callback. Rather than drop it on the floor,
# pass it along, unless it's just the stream being closed.
return isinstance(value, StreamClosedError)
def get_groups_request_handler(request):
mock_globus_groups_response = [
{
'id': '21c6bc5d-fc12-4f60-b999-76766cd596c2',
'my_memberships': [{
'role': 'manager'
}],
},
{
'id': '915dcd61-c842-4ea4-97c6-57396b936016',
'my_memberships': [{
'role': 'member'
}],
},
{
'id': 'd11abe71-5132-4c04-a4ad-50926885dc8c',
'my_memberships': [{
'role': 'member',
}],
},
]
assert request.method == 'GET', request.method
resp = BytesIO(json.dumps(mock_globus_groups_response).encode('utf-8'))
return HTTPResponse(
request=request,
code=200,
headers={'Content-Type': 'application/json'},
buffer=resp,
)
def _on_timeout(self):
self._timeout = None
self._run_callback(
HTTPResponse(self.request,
599,
request_time=time.time() - self.start_time,
error=HTTPError(599, "Timeout")))
self.stream.close()
def test_str(self):
response = HTTPResponse(HTTPRequest('http://example.com'),
200,
headers={},
buffer=BytesIO())
s = str(response)
self.assertTrue(s.startswith('HTTPResponse('))
self.assertIn('code=200', s)
def cleanup(self):
try:
yield
except Exception, e:
logging.warning("uncaught exception", exc_info=True)
self._run_callback(HTTPResponse(self.request, 599, error=e,
request_time=time.time() - self.start_time,
))
