def _handle_exception(self, typ, value, tb):
if self.final_callback:
self._remove_timeout()
if isinstance(value, StreamClosedError):
value = HTTPError(599, "Stream closed")
self._run_callback(
HTTPResponse(
self.request,
599,
error=value,
request_time=self.io_loop.time() - self.start_time,
))
if hasattr(self, "stream"):
# TODO: this may cause a StreamClosedError to be raised
# by the connection's Future. Should we cancel the
# connection more gracefully?
self.stream.close()
return True
else:
# If our callback has already been called, we are probably
# catching an exception that is not caused by us but rather
# some child of our callback. Rather than drop it on the floor,
# pass it along, unless it's just the stream being closed.
return isinstance(value, StreamClosedError)
async def execute_downstream(self) -> HTTPResponse:
"""
Executes the downstream request (Jupyter to AWS service) and return the response or the error
after adding SigV4 authentication.
"allow_nonstandard_methods" is used because Tornado rejects POST requests without a body without this parameter,
and some operations send such requests (such as S3.InitiateMultipartUpload)
:return: the HTTPResponse
"""
if (self.whitelisted_services is not None
and self.service_info.service_name
not in self.whitelisted_services):
raise HTTPError(
403,
message=
f"Service {self.service_info.service_name} is not whitelisted for proxying requests",
)
downstream_request_path = self.upstream_request.path[len("/awsproxy"
):] or "/"
return await AsyncHTTPClient().fetch(
HTTPRequest(
method=self.upstream_request.method,
url=self._compute_downstream_url(downstream_request_path),
headers=self._compute_downstream_headers(
downstream_request_path),
body=self.upstream_request.body or None,
follow_redirects=False,
allow_nonstandard_methods=True,
))
def put(self, id):
'''
user : change its own info [cname]
admin : change anyuser`s info [cname, nrole]
'''
if self.current_user['nrole'] == 0: # user role
''' normal user can only change itself '''
if self.current_user['nid'] != id:
raise HTTPError(403)
#user = json.loads(self.request.body)
#nid, cname = user
nid = self.get_argument("nid")
cname = self.get_argument("cname")
cpassword = self.get_argument("cpassword")
if nid != id:
raise HTTPError(403)
cursor = self.conn.cursor()
''' check the id ?'''
try:
cursor.execute("update users set cname={0}, cpassword={1} \
where nid={2}".format(cname, cpassword, nid))
except:
raise HTTPError(500)
self.conn.commit()
cursor.close()
return
else: # admin role
#user = json.loads(self.request.body)
#nid, cname, nrole = user
nid = self.get_argument("nid")
cname = self.get_argument("cname")
cpassword = self.get_argument("cpassword")
nrole = self.get_argument("nrole")
cursor = self.conn.cursor()
''' check the id ?'''
try:
cursor.execute("update users set cname={0}, cpassword={1}, nrole={2} \
where nid={3}".format(cname, cpassword, nrole, nid))
except:
raise HTTPError(500)
self.conn.commit()
cursor.close()
return
def get_with_retries_v2(self,
url,
websocket=None,
max_retry_times=3,
as_file=False):
"""
Use this function.
"""
try:
#print("started get_with_retries")
request = self.simple_request(url)
http_client = AsyncHTTPClient()
response = yield http_client.fetch(request)
result = Result(response, as_file)
except HTTPError as e:
msg = "HTTP Exception get_with_retries_v2:{0}".format(e)
self.printf("{0} {1}".format(e.code, msg))
try:
self.printf("TrackingId:{0}".format(
e.response.headers.get("Trackingid")))
except Exception as te:
self.printf("No TrackingId.")
try:
try:
msg = json.loads(e.response.body.decode('utf8'))
except Exception as ex:
msg = e.response.body.decode('utf8')
except Exception as exx:
pass #probably a 599 timeout
#self.printf("New msg: {0}".format(msg))
if (e.code in [400, 409, 429]
or e.code >= 500) and max_retry_times > 0:
if e.code == 429:
retry_after = None
try:
retry_after = e.response.headers.get("Retry-After")
except Exception as e:
pass
if retry_after == None:
retry_after = 30
else:
retry_after = 10
msg = "{0} hit, waiting for {1} seconds and then retrying...".format(
e.code, retry_after)
self.printf(msg)
if websocket != None:
update_obj = {
"resource": "http_error",
"message": msg,
"update": True
}
websocket.write_message(json.dumps(update_obj))
yield tornado.gen.sleep(int(retry_after))
max_retry_times -= 1
result = yield self.get_with_retries(url, websocket,
max_retry_times)
else:
raise HTTPError(e.code, response=e.response) from e
raise tornado.gen.Return(result)
def _unicode(s):
if isinstance(s, str):
try:
return s.decode("utf-8")
except UnicodeDecodeError:
raise HTTPError(400, "Non-utf8 argument")
# assert isinstance(s, unicode)
return s
def _on_close(self):
if self.callback is not None:
callback = self.callback
self.callback = None
callback(
HTTPResponse(self.request,
599,
error=HTTPError(599, "Connection closed")))
def _get_token(url: str) -> str:
app_log.info("url: %s", url)
parts: Tuple[str, ...] = tuple(
url.split(sep="/login?next=%2Fhub%2Fhome&token=", maxsplit=1))
if len(parts) != 2:
app_log.error("url:\n%s", url)
raise HTTPError(500, f"url: {url}")
return parts[1]
def _on_timeout(self, key):
request, callback, timeout_handle = self.waiting[key]
self.queue.remove((key, request, callback))
timeout_response = HTTPResponse(
request, 599, error=HTTPError(599, "Timeout"),
request_time=self.io_loop.time() - request.start_time)
self.io_loop.add_callback(callback, timeout_response)
del self.waiting[key]
def test_fail_303(self, HTTPClient):
self.setup_httpclient(HTTPClient)
self.mock_httpclient.fetch.side_effect = HTTPError(
303, response="303 response")
self.assertEqual(homestead._sync_http_request("http://homestead/ping"),
"303 response")
self.mock_httpclient.fetch.assert_called_once_with(
"http://homestead/ping", follow_redirects=False, allow_ipv6=True)
def test_request_response_error(self):
self.transport._http = self.http_mock
self.http_mock.fetch.side_effect = HTTPError(code=404)
with self.assertRaises(TTransportException):
yield self.transport.request(FContext(), bytearray([0, 0, 0, 1,
0]))
def get(self):
by = self.get_argument('by')
if by == 'last':
status = Status.objects()[0]
status.get_brief_status()
self.write(status)
else:
raise HTTPError(400)
def test_fail_http_exception(self, HTTPClient):
self.setup_httpclient(HTTPClient)
e = HTTPError(503)
self.mock_httpclient.fetch.side_effect = e
self.assertEqual(homestead._sync_http_request("http://homestead/ping"),
e)
self.mock_httpclient.fetch.assert_called_once_with(
"http://homestead/ping", follow_redirects=False, allow_ipv6=True)
def _location(httpresponse):
"""Retrieves the Location header from this HTTP response,
throwing a 500 error if it is missing"""
if httpresponse.headers.get_list('Location'):
return httpresponse.headers.get_list('Location')[0]
else: # pragma: no cover
_log.error("Could not retrieve Location header from HTTPResponse %s" % httpresponse)
raise HTTPError(500)
def _on_timeout(self):
self._timeout = None
self._run_callback(
HTTPResponse(self.request,
599,
request_time=time.time() - self.start_time,
error=HTTPError(599, "Timeout")))
self.stream.close()
def post(self):
#good_json = json.loads(self.request.body)
#good = good_json.get("good")
#good = json.loads(self.request.body)
cname = self.get_argument('cname')
dprice = self.get_argument('dprice')
cdesc = self.get_argument('cdesc')
ncategoryid = self.get_argument('ncategoryid')
nount = self.get_argument('ncount')
cursor = self.conn.cursor()
''' check available is omitted '''
try:
cursor.execute("insert into tbgoods (cname, dprice, cdesc, ncategoryid, ncount) \
values ('{0}', {1}, '{2}', {3}, {4}) returning nid".format(cname, dprice, cdesc, ncategoryid, ncount))
except:
raise HTTPError(500)
nid = cursor.fetchone()
self.conn.commit()
try:
cursor.execute("select nid, cname, dprice, cdesc, ncategoryid, ncount \
from tbgoods where nid={0}".format(nid))
except:
raise HTTPError(500)
if cursor.rowcount > 0:
goodinfo = cursor.fetchone()
cursor.close()
# return self.write(json_encode({
# "good":{
# 'nid':goodinfo[0],
# 'cname':goodinfo[1],
# 'dprice':goodinfo[2],
# 'cdesc':goodinfo[3],
# 'ncategoryid':goodinfo[4],
# 'ncount':goodinfo[5]}
# }))
return self.write(json.dumps({
'nid':goodinfo[0],
'cname':goodinfo[1],
'dprice':goodinfo[2],
'cdesc':goodinfo[3],
'ncategoryid':goodinfo[4],
'ncount':goodinfo[5]
}))
async def get(self):
log.debug("%s received login request" % type(self).__name__)
if config.oauth_redirect_uri:
redirect_uri = config.oauth_redirect_uri
else:
redirect_uri = "{0}://{1}".format(self.request.protocol,
self.request.host)
params = {
'redirect_uri': redirect_uri,
'client_id': config.oauth_key,
}
# Some OAuth2 backends do not correctly return code
next_arg = self.get_argument('next', None)
url_state = self.get_argument('state', None)
code = self.get_argument('code', extract_urlparam('code', next_arg))
url_state = self.get_argument('state',
extract_urlparam('state', next_arg))
# Seek the authorization
cookie_state = self.get_state_cookie()
if code:
if cookie_state != url_state:
log.warning("OAuth state mismatch: %s != %s", cookie_state,
url_state)
raise HTTPError(400, "OAuth state mismatch")
# For security reason, the state value (cross-site token) will be
# retrieved from the query string.
params.update({
'client_secret': config.oauth_secret,
'code': code,
'state': url_state
})
user = await self.get_authenticated_user(**params)
if user is None:
raise HTTPError(403)
log.debug("%s authorized user, redirecting to app." %
type(self).__name__)
self.redirect('/')
else:
# Redirect for user authentication
state = uuid.uuid4().hex
params['state'] = state
self.set_state_cookie(state)
await self.get_authenticated_user(**params)
def delete(self, id):
cursor = self.conn.cursor()
try:
cursor.execute("delete from tbgoods where nid={0}".format(id))
except:
raise HTTPError(500)
self.conn.commit()
cursor.close()
return
def on_connection_close(self):
if self.final_callback is not None:
message = "Connection closed"
if self.stream.error:
raise self.stream.error
try:
raise HTTPError(599, message)
except HTTPError:
self._handle_exception(*sys.exc_info())
def test_can_approve_no_service(self):
with patch.object(Service,
'get',
side_effect=couch.NotFound(
HTTPError(404, 'Not Found'))):
user = User(**USER)
func = partial(self.repo.can_approve, user)
result = IOLoop.instance().run_sync(func)
assert result is False
def delete_checkpoint(self, checkpoint_id, path):
"""delete a file's checkpoint"""
cp_path = self.checkpoint_path(checkpoint_id, path)
fs_cp_path = self._to_fs_path(cp_path)
if not self.fs.exists(fs_cp_path):
raise HTTPError(404, f'Checkpoint does not exist: {path}@{checkpoint_id}')
self.log.info("Removing checkpoint %s", cp_path)
with self.perm_to_403():
self.fs.delete(fs_cp_path)
def request(body_type,
namespace,
payload,
response_type=None,
content_type=None):
"""
Makes request to blazegraph
:param body_type: type of request (e.g: 'query', 'update' or None)
:param payload: request body
:param namespace: namespace of request
:param response_type: Accept type for header ('csv', 'xml', 'json' or None)
:param content_type: Content type for header
:return: Response from blazegraph
"""
if not namespace:
raise HTTPError(400, "Namespace or repository missing")
db_url = _namespace_url(namespace)
headers = _set_accept_header(response_type)
if content_type:
headers['Content-Type'] = content_type
if body_type:
body = urlencode({body_type: payload})
else:
body = payload
logging.debug('request. body_type:' + str(body_type))
logging.debug('request. payload:' + str(payload))
try:
rsp = yield AsyncHTTPClient().fetch(db_url,
method="POST",
body=body,
headers=headers)
except HTTPError as exc:
# If response is 404 then namespace does not exist.
# Lazily create namespace and retry request.
if exc.code == 404:
try:
yield create_namespace(namespace)
except HTTPError as exc:
# Namespace already exists with a 409 error
if exc.code != 409:
logging.error("%s:%s" % (str(exc.code), exc.message))
else:
logging.warning('Namespace %s already exists' % namespace)
rsp = yield AsyncHTTPClient().fetch(db_url,
method="POST",
body=body,
headers=headers)
else:
raise exc
raise Return(rsp)
def test_fetch_returns_none_when_request_fails(self):
uri = 'http://dbpedia.com/resource/Sample'
client = Mock()
client.fetch = Mock(side_effect=HTTPError(500))
self.summarum_endpoint = Endpoint(http_client=client)
response = yield self.summarum_endpoint.fetch(uri)
self.assertEquals(response, None)
def _on_timeout(self):
self._timeout = None
if self.callback is not None:
self.callback(
HTTPResponse(self.request,
599,
error=HTTPError(599, "Timeout")))
self.callback = None
self.stream.close()
async def test_server_reponse_599(self, mock_log, http_client):
http_client.instance().head.side_effect = HTTPError(code=403,
response=None)
self.task.store_vulnerability = MagicMock()
result = await self.task()
expected = None
self.assertEqual(result, expected)
def get(self, path):
"""
Proxy API requests to GitHub, adding authentication parameter(s) if
they have been set.
"""
# Get access to the notebook config object
c = GitHubConfig(config=self.config)
try:
query = self.request.query_arguments
params = {key: query[key][0].decode() for key in query}
api_path = url_path_join(c.api_url, url_escape(path))
params['per_page'] = 100
access_token = params.pop('access_token', None)
if access_token and c.allow_client_side_access_token == True:
token = access_token
elif access_token and c.allow_client_side_access_token == False:
msg = ("Client side (JupyterLab) access tokens have been "
"disabled for security reasons.\nPlease remove your "
"access token from JupyterLab and instead add it to "
"your notebook configuration file:\n"
"c.GitHubConfig.access_token = '<TOKEN>'\n")
raise HTTPError(403, msg)
elif c.access_token != '':
# Preferentially use the config access_token if set
token = c.access_token
api_path = url_concat(api_path, params)
client = AsyncHTTPClient()
request = HTTPRequest(
api_path,
validate_cert=c.validate_cert,
user_agent='JupyterLab GitHub',
headers={"Authorization": "token {}".format(token)})
response = yield client.fetch(request)
data = json.loads(response.body.decode('utf-8'))
# Check if we need to paginate results.
# If so, get pages until all the results
# are loaded into the data buffer.
next_page_path = self._maybe_get_next_page_path(response)
while next_page_path:
request = copy.copy(request)
request.url = next_page_path
response = yield client.fetch(request)
next_page_path = self._maybe_get_next_page_path(response)
data.extend(json.loads(response.body.decode('utf-8')))
# Send the results back.
self.finish(json.dumps(data))
except HTTPError as err:
self.set_status(err.code)
message = err.response.body if err.response else str(err.code)
self.finish(message)
async def test_server_reponse_403_logging(self, mock_log, http_client):
request = HTTPRequest(url='url')
response = HTTPResponse(code=403, request=request)
http_client.instance().head.side_effect = HTTPError(code=403,
response=response)
self.task.store_vulnerability = MagicMock()
await self.task()
self.assertTrue(mock_log.warning.called)
def check_access(handler, roleneed):
if isinstance(roleneed,MasterRoleNeed):
return
if not roleneed:
raise HTTPError(403)
checkname = handler.__checkname__
if handler.__needcheck__.get('url',None):
if not checkname in roleneed.nodes:
raise HTTPError(403)
ctx_params = handler.__needcheck__.get('ctx_param',None)
if ctx_params:
for ctx_param in ctx_params.split(','):
ctx_val = handler.get_argument(ctx_param, handler.path_kwargs.get(ctx_param,None))
if not ctx_val in roleneed.ctx_vals.get(ctx_param,()):
raise HTTPError(403)
def delete(version_key):
""" Stops all instances on this machine for a version.
Args:
version_key: A string specifying a version key.
"""
try:
yield stop_app(version_key)
except BadConfigurationException as error:
raise HTTPError(HTTPCodes.BAD_REQUEST, error.message)
def test_request_timeout(self):
self.transport._http = self.http_mock
self.http_mock.fetch.side_effect = HTTPError(code=599)
with self.assertRaises(TTransportException) as cm:
yield self.transport.request(FContext(), bytearray([0, 0, 0, 1,
0]))
self.assertEqual(TTransportExceptionType.TIMED_OUT, cm.exception.type)
self.assertEqual("request timed out", cm.exception.message)
def delete(self, id):
cursor = self.conn.cursor()
''' check the id first? '''
try:
cursor.execute("delete from tbcategories where nid={0}".format(id))
except:
raise HTTPError(500, "db error")
self.conn.commit()
cursor.close()
return
def handle_exception(self, typ, error, tb):
if isinstance(error, _RequestTimeout):
if self._stream_ended:
self.finish()
return True
else:
error = HTTPError(599, "Timeout")
self._remove_timeout()
self._unregister_unfinished_streams()
if hasattr(self, "stream_id"):
self.context.remove_stream_delegate(self.stream_id)
# FIXME: our nginx server will simply reset stream,
# without increase the window size which consumed by
# queued data frame which was belongs to the stream we're resetting
# self.context.reset_stream(self.stream_id, flush=True)
self.context.reset_stream_ids.append(self.stream_id)
error.__traceback__ = tb
response = HTTP2Response(self.request, 599, error=error, request_time=self.io_loop.time() - self.start_time)
self._run_callback(response)
return True
def __init__(self, errno, message):
HTTPError.__init__(self, 599, message)
self.errno = errno
def __init__(self, status_code, user_message, log_message=None, *args):
HTTPError.__init__(self, status_code, log_message, *args)
self.user_message = user_message
def __init__(self, errno: int, message: str) -> None:
HTTPError.__init__(self, 599, message)
self.errno = errno
