async def httpcat(self, ctx: commands.Context, code=999) -> None:
"""Sends an embed with an image of a cat, portraying the status code.
If the status code is wrong than it will return a random status cat.
If no status code given it will return a random status cat."""
embed = discord.Embed(title=f'**Status: {code}**')
embed.set_footer(text=f"Image got from https://http.cat/{code}.jpg.")
try:
HTTPStatus(code)
except ValueError:
# choosing a random valid status code
code = random.choice(VALID_CODE)
# Over writing the previous footer and title
embed = discord.Embed(title=f'**Status: {code}**')
embed.set_footer(
text=
f"""Inputted status code does not exist. Here is a random status.
Image got from https://http.cat/{code}.jpg."""
)
HTTPStatus(code)
embed.set_image(url=f'https://http.cat/{code}.jpg')
else:
embed.set_image(url=f'https://http.cat/{code}.jpg')
finally:
await ctx.send(embed=embed)
def test_enteryourcode_post_invalid_does_not_redirect(
self, code_form_mock, is_otp_verified_by_pds_mock, _):
""" Test enteryourcode page posts to does not redirect for
invalid otp """
test_cases = (
(66666, constants.INCORRECT_OTP_MAX_RETRIES),
(23456, constants.INCORRECT_OTP),
(11234, constants.INCORRECT_OTP_MAX_RETRIES),
(None, constants.INCORRECT_OTP_MAX_RETRIES),
(0, constants.INCORRECT_OTP),
('string', constants.INCORRECT_OTP_MAX_RETRIES),
(32000, constants.INCORRECT_OTP),
)
form_mock = MagicMock()
add_incorrect_otp_error_mock = MagicMock()
form_mock.add_incorrect_otp_error = add_incorrect_otp_error_mock
code_form_mock.return_value = form_mock
for case in test_cases:
with self.subTest(case=case):
number, status_result = case
is_otp_verified_by_pds_mock.return_value = status_result
result = self.client.post('/enteryourcode',
data=dict(enterOtpInput=str(number)))
if status_result == constants.INCORRECT_OTP:
assert add_incorrect_otp_error_mock.called
assert HTTPStatus(result.status_code) == HTTPStatus.OK
else:
assert HTTPStatus(result.status_code) == HTTPStatus.FOUND
assert '/incorrectcodeerror' in result.headers['Location']
def _fetch_payloads(self, request_url: str) -> Generator:
"""
Recursively generates payloads from paginated responses
"""
request = Request(request_url)
try:
response = urlopen(request, timeout=30)
except urllib.error.HTTPError as e:
status: HTTPStatus = HTTPStatus(e.code)
yield 'Failed to load remote: "%s" (%s %s)' % (request_url, e.code,
status.phrase)
sys.exit(1)
if response.status != 200:
status: HTTPStatus = HTTPStatus(response.status)
yield 'Failed to load remote: "%s" (%s %s)' % (
request_url, response.status, status.phrase)
sys.exit(1)
payload: dict = json.loads(response.read().decode('utf-8'))
yield payload
if 'next' in payload:
yield from self._fetch_payloads(payload['next'])
def set_preference(user_details, session_id):
app.logger.info("setting preference")
headers = {"Content-type": "application/json"}
cookies = dict(session_id=session_id)
user_json = json.dumps({"preference": user_details.preference})
try:
preference_result_request = requests.post(
url=app.config["SET_PREFERENCE_URL"],
headers=headers,
data=user_json,
cookies=cookies)
preference_result_request.raise_for_status()
if HTTPStatus(preference_result_request.status_code) is HTTPStatus.OK:
return True
except requests.exceptions.RequestException as e:
if isinstance(e, requests.exceptions.HTTPError) \
and HTTPStatus(e.response.status_code) == HTTPStatus.FORBIDDEN:
app.logger.info(
"set_preference: {status_code: 403, status: FORBIDDEN}")
return False
app.logger.info("preference not set successfully")
return False
def get_store_preference_result(session_id):
app.logger.info("storing preference result")
headers = {"Content-type": "application/json"}
cookies = dict(session_id=session_id)
try:
preference_result_request = requests.get(
url=app.config["SET_PREFERENCE_RESULT_URL"],
headers=headers,
cookies=cookies)
preference_result_request.raise_for_status()
app.logger.info("store_preference_result",
{"status_code": preference_result_request.status_code})
if HTTPStatus(preference_result_request.status_code) is HTTPStatus.OK:
return "success"
if HTTPStatus(preference_result_request.status_code
) is HTTPStatus.PARTIAL_CONTENT:
return "not_completed"
except requests.exceptions.RequestException as e:
app.logger.info("storing preference exception")
return "failure"
app.logger.info("storing preference result failure")
return "failure"
def get_current_preference(session_id):
"""retunrs False if not successful """
app.logger.info("getting current preference")
headers = {"Content-type": "application/json"}
cookies = dict(session_id=session_id)
try:
preference_result_request = requests.get(
url=app.config["PREFERENCE_RESULT_URL"],
headers=headers,
cookies=cookies)
preference_result_request.raise_for_status()
if HTTPStatus(preference_result_request.status_code) is HTTPStatus.OK:
result = preference_result_request.json()
if result.get("get_preference_result"
) == constants.GET_PREFERENCE_SUCCESS:
if result.get("opted_out") in ('active', 'inactive'):
return result.get("opted_out")
else:
return constants.GET_PREFERENCE_EMPTY
elif HTTPStatus(preference_result_request.status_code
) is HTTPStatus.PARTIAL_CONTENT:
return constants.GET_PREFERENCE_INCOMPLETE
except requests.exceptions.RequestException:
return constants.GET_PREFERENCE_FAILURE
return constants.GET_PREFERENCE_EMPTY
def CreateDescriptor():
with open("Descriptor.xml") as xml:
payload = xml.read()
r = requests.post(GSCLURIhead + "/applications/Thermometer/containers",
auth=BasicAuth,
data=payload)
app.logger.info(str(HTTPStatus(r.status_code)))
r = requests.post(GSCLURIhead + "/applications/Fan/containers",
auth=BasicAuth,
data=payload)
app.logger.info(str(HTTPStatus(r.status_code)))
with open("DescriptorThermometer.xml") as xml:
payload = xml.read()
r = requests.post(
GSCLURIhead +
"/applications/Thermometer/containers/DESCRIPTOR/contentInstances",
auth=BasicAuth,
data=payload)
app.logger.info(str(HTTPStatus(r.status_code)))
with open("DescriptorFan.xml") as xml:
payload = xml.read()
r = requests.post(
GSCLURIhead +
"/applications/Fan/containers/DESCRIPTOR/contentInstances",
auth=BasicAuth,
data=payload)
app.logger.info(str(HTTPStatus(r.status_code)))
return "200"
def _session_get(self, url: str, **kwargs: Any) -> requests.Response:
if not getenv("NASTY_DISRESPECT_ROBOTSTXT"):
global crawl_delay
if crawl_delay is None:
response = self._session.get(
"https://mobile.twitter.com/robots.txt")
for line in response.text.splitlines():
if line.lower().startswith("crawl-delay:"):
crawl_delay = float(line[len("crawl-delay:"):])
break
else:
raise RuntimeError("Could not determine crawl-delay.")
logger.debug(" Determined crawl-delay of {:.2f}s.".format(
crawl_delay))
sleep(crawl_delay)
response = self._session.get(url, **kwargs)
status = HTTPStatus(response.status_code)
logger.debug(" Received {} {} for {}".format(
status.value, status.name, response.url))
if response.status_code != HTTPStatus.OK.value:
raise UnexpectedStatusCodeException(
response.url, HTTPStatus(response.status_code))
return response
async def process_request(
path: str, _: list
) -> Optional[Tuple[HTTPStatus, Iterable[Tuple[str, str]], bytes]]:
if path == '/ws':
return None
elif path == '/':
return HTTPStatus(200), [], b''
else:
return HTTPStatus(404), [], b''
async def handle_org_setup(self, req: object,
params: Dict[str, str]) -> HTTPResponse:
org_id = params['org_id']
if org_id in self.orgs:
return (HTTPStatus(304), {}, bytes())
print(f'setting up org {org_id}')
org = await Organization.create(org_id)
self.orgs[org_id] = org
return (HTTPStatus(201), {}, bytes())
async def handle_stream_teardown(self, req: object,
params: Dict[str, str]) -> HTTPResponse:
stream_id = params['stream_id']
if stream_id not in self.streams:
return (HTTPStatus(404), {}, bytes())
print(f'tearing down stream {stream_id}')
stream = self.streams[stream_id]
stream.deleted = True
return (HTTPStatus(200), {}, bytes())
def parse_error(self, error_body):
try:
json_error = json.loads(error_body)
code = HTTPStatus(int(json_error['code']))
return code, json_error
except json.decoder.JSONDecodeError as e:
LOGGER.error("Decoder exception loading error msg: %s;"
"%s", error_body, str(e))
return HTTPStatus(500), {"message": error_body}
async def handle_org_teardown(self, req: object,
params: Dict[str, str]) -> HTTPResponse:
org_id = params['org_id']
if org_id not in self.orgs:
return (HTTPStatus(404), {}, bytes())
org = self.orgs[org_id]
print(f'tearing down org {org_id}')
await org.close()
await db.delete_emotes(org_id)
return (HTTPStatus(200), {}, bytes())
def _get_header_lines(self, version):
result = []
phrase = HTTPStatus(self.status).phrase
result.append(b'HTTP/%s %i %s' %
(version.encode(), self.status, phrase.encode()))
for header, value in self.headers.items():
result.append(b'%s: %s' % (header.encode(), value.encode()))
result.append(b'\r\n')
return b'\r\n'.join(result)
def assert_on_web_request_end(self, mon, time, app, response, expected_status):
request = Mock(name="request")
view = Mock(name="view")
other_time = 156.9
state = {"time_start": other_time}
mon.on_web_request_end(app, request, response, state, view=view)
assert state["time_end"] == time()
assert state["latency_end"] == time() - other_time
assert state["status_code"] == HTTPStatus(expected_status)
assert mon.http_response_latency[-1] == time() - other_time
assert mon.http_response_codes[HTTPStatus(expected_status)] == 1
def invoke(self, invocation_context: ApiInvocationContext) -> Response:
passthrough_behavior = invocation_context.integration.get(
"passthroughBehavior") or ""
request_template = invocation_context.integration.get(
"requestTemplates",
{}).get(invocation_context.headers.get(HEADER_CONTENT_TYPE))
# based on the configured passthrough behavior and the existence of template or not,
# we proceed calling the integration or raise an exception.
try:
self.check_passthrough_behavior(passthrough_behavior,
request_template)
except MappingTemplates.UnsupportedMediaType:
http_status = HTTPStatus(415)
return MockIntegration._create_response(
http_status.value,
headers={"Content-Type": APPLICATION_JSON},
data=json.dumps({"message": f"{http_status.phrase}"}),
)
# request template rendering
request_payload = self.request_templates.render(invocation_context)
# mapping is done based on "statusCode" field
status_code = 200
if invocation_context.headers.get(
HEADER_CONTENT_TYPE) == APPLICATION_JSON:
try:
mock_response = json.loads(request_payload)
status_code = mock_response.get("statusCode", status_code)
except Exception as e:
LOG.warning(
"failed to deserialize request payload after transformation: %s",
e)
http_status = HTTPStatus(500)
return MockIntegration._create_response(
http_status.value,
headers={"Content-Type": APPLICATION_JSON},
data=json.dumps({"message": f"{http_status.phrase}"}),
)
# response template
response = MockIntegration._create_response(status_code,
invocation_context.headers,
data=request_payload)
response._content = self.response_templates.render(invocation_context,
response=response)
# apply response parameters
response = self.apply_response_parameters(invocation_context, response)
if not invocation_context.headers.get(HEADER_CONTENT_TYPE):
invocation_context.headers.update(
{HEADER_CONTENT_TYPE: APPLICATION_JSON})
return response
def CreateData():
with open("Data.xml") as xml:
payload = xml.read()
r = requests.post(GSCLURIhead + "/applications/Thermometer/containers",
auth=BasicAuth,
data=payload)
app.logger.info(str(HTTPStatus(r.status_code)))
r = requests.post(GSCLURIhead + "/applications/Fan/containers",
auth=BasicAuth,
data=payload)
app.logger.info(str(HTTPStatus(r.status_code)))
return "200"
def StatusDescription():
try:
statusCode = int(request.args.get('statusCode'))
except:
return HTTPStatus(400).description
try:
response = HTTPStatus(statusCode).description
except:
response = HTTPStatus(400).description
return response
def assert_on_web_request_end(self, mon, time, app, response,
expected_status):
request = Mock(name='request')
view = Mock(name='view')
other_time = 156.9
state = {'time_start': other_time}
mon.on_web_request_end(app, request, response, state, view=view)
assert state['time_end'] == time()
assert state['latency_end'] == time() - other_time
assert state['status_code'] == HTTPStatus(expected_status)
assert mon.http_response_latency[-1] == time() - other_time
assert mon.http_response_codes[HTTPStatus(expected_status)] == 1
def CreateApp():
with open("ApplicationThermometer.xml") as xml:
payload = xml.read()
r = requests.post(GSCLURIhead + "/applications",
auth=BasicAuth,
data=payload)
app.logger.info(str(HTTPStatus(r.status_code)))
with open("ApplicationFan.xml") as xml:
payload = xml.read()
r = requests.post(GSCLURIhead + "/applications",
auth=BasicAuth,
data=payload)
app.logger.info(str(HTTPStatus(r.status_code)))
return "200"
def check_status_of_pds_search_result(session_id):
search_result = ""
headers = {"Content-type": "application/json"}
cookies = dict(session_id=session_id)
try:
app.logger.info("submitting pds search result request")
pds_search_result_request = requests.get(
url=app.config["PDS_SEARCH_RESULT_URL"],
headers=headers,
cookies=cookies,
timeout=app.config["PDS_REQUEST_TIMEOUT"],
)
pds_search_result_request.raise_for_status()
if HTTPStatus(pds_search_result_request.status_code) is HTTPStatus.OK:
result = pds_search_result_request.json()
# TODO: Remove functional side effects of writing to session
if result.get("sms"):
session["sms"] = result.get("sms")
if result.get("email"):
session["email"] = result.get("email")
search_result = result.get("search_result")
app.logger.info("pds search result response received",
{"search_result": search_result})
return search_result
except requests.exceptions.RequestException as exc:
if isinstance(exc, requests.exceptions.Timeout):
return constants.PDS_REQUEST_TIMEOUT
if isinstance(exc, requests.exceptions.HTTPError) \
and HTTPStatus(exc.response.status_code) == HTTPStatus.UNAUTHORIZED:
return "invalid_user"
if isinstance(exc, requests.exceptions.HTTPError) \
and HTTPStatus(exc.response.status_code) == HTTPStatus.UNPROCESSABLE_ENTITY:
return "insufficient_data"
if isinstance(exc, requests.exceptions.HTTPError) \
and HTTPStatus(exc.response.status_code) == HTTPStatus.NOT_ACCEPTABLE:
return "age_restriction_error"
exception_type = type(exc).__name__
status_code = exc.response.status_code or "not_applicable"
log_safe_exception(exc)
raise NDOP_RequestError(
f'pds search result request failed, original exception type {exception_type}, response status code {status_code}'
)
def _handle_bad_response(response: Response):
try:
error = response.json()
raise OAuthError(HTTPStatus(response.status_code),
error.get('error'),
error.get('error_description'))
except BaseException as ex:
if type(ex) != OAuthError:
_logger.exception(
'_handle_bad_response - error while getting error as json - %s - %s'
% (type(ex), str(ex)))
raise OAuthError(HTTPStatus(response.status_code),
'unknown_error', response.text)
else:
raise
async def handle_exception(self, exc):
"""Handle an HTTP exception.
:param exc: The exception object to handle, which should be a subclass
of `unit.exceptions.HTTPException` class.
"""
assert hasattr(exc, 'code')
code = exc.code
assert code in message.HTTP_STATUS_CODES
# Try to find an exception handler
handler = self._exception_handlers.get(code)
if handler is not None:
try:
result = handler(request=self._request)
if inspect.isawaitable(result):
result = await result
self._response = await self.make_response(result)
return
except:
if self._debug:
raise
# Handle redirect exception
if isinstance(exc, exceptions.RedirectException):
assert exc.new_url is not None
headers = [('Location', exc.new_url)]
self._response = await self.make_response((None, code, headers))
return
# Generate an error message using default error message template
if exc.phrase is not None:
phrase = exc.phrase
else:
phrase = HTTPStatus(code).phrase
if exc.description is not None:
description = exc.description
else:
description = HTTPStatus(code).description
error_message = self.error_message_format % {
'code': code,
# HTML encode to prevent Cross Site Scripting attacks
'message': html.escape(phrase, quote=False),
'explain': html.escape(description, quote=False)
}
self._response = await self.make_response((error_message, code))
def get_body(self, environ=None):
code = getattr(self, "code", 500)
return render_template(self.template,
**self.template_kws,
response_code=code,
response_message=HTTPStatus(code).phrase)
def problems_middleware(request, handler):
try:
response = yield from handler(request)
except ProblemException as exc:
response = problem(status=exc.status, detail=exc.detail, title=exc.title,
type=exc.type, instance=exc.instance, headers=exc.headers, ext=exc.ext)
except (werkzeug_HTTPException, _HttpNotFoundError) as exc:
response = problem(status=exc.code, title=exc.name, detail=exc.description)
except web.HTTPError as exc:
if exc.text == "{}: {}".format(exc.status, exc.reason):
detail = HTTPStatus(exc.status).description
else:
detail = exc.text
response = problem(status=exc.status, title=exc.reason, detail=detail)
except (
web.HTTPException, # eg raised HTTPRedirection or HTTPSuccessful
asyncio.CancelledError, # skipped in default web_protocol
):
# leave this to default handling in aiohttp.web_protocol.RequestHandler.start()
raise
except asyncio.TimeoutError as exc:
# overrides 504 from aiohttp.web_protocol.RequestHandler.start()
logger.debug('Request handler timed out.', exc_info=exc)
response = _generic_problem(HTTPStatus.GATEWAY_TIMEOUT, exc)
except Exception as exc:
# overrides 500 from aiohttp.web_protocol.RequestHandler.start()
logger.exception('Error handling request', exc_info=exc)
response = _generic_problem(HTTPStatus.INTERNAL_SERVER_ERROR, exc)
if isinstance(response, ConnexionResponse):
response = yield from AioHttpApi.get_response(response)
return response
def test_landingpage_get(self):
""" Test landing page URL returns 200 """
result = self.client.get('/landingpage')
assert HTTPStatus(result.status_code) == HTTPStatus.OK
assert 'ndop_seen_cookie_message' in result.headers['Set-Cookie']
def test_root_redirect(self):
""" Test root URL gives a 302 """
result = self.client.get('/')
assert HTTPStatus(result.status_code) == HTTPStatus.FOUND
assert routes.get_absolute(
"main.landing_page") in result.headers['Location']
async def _send_http_reply(
self,
stream: Stream,
conn: h11.Connection,
status_code: int,
headers: Dict[bytes, bytes] = {},
data: Optional[bytes] = None,
) -> None:
reason = HTTPStatus(status_code).phrase
headers = list({
**headers,
# Add default headers
b"server": self.server_header,
b"date": format_date_time(None).encode("ascii"),
b"content-Length": str(len(data or b"")).encode("ascii"),
# Inform we don't support keep-alive (h11 will know what to do from there)
b"connection": b"close",
}.items())
try:
await stream.send_all(
conn.send(
h11.Response(status_code=status_code,
headers=headers,
reason=reason)))
if data:
await stream.send_all(conn.send(h11.Data(data=data)))
await stream.send_all(conn.send(h11.EndOfMessage()))
except trio.BrokenResourceError:
# Given we don't support keep-alive, the connection is going to be
# shutdown anyway, so we can safely ignore the fact peer has left
pass
def describe_response(
status: typing.Union[int, HTTPStatus],
description: str = "",
*,
content: typing.Union[typing.Type[BaseModel], dict] = None,
headers: dict = None,
links: dict = None,
) -> typing.Callable[[T], T]:
"""
describe a response in HTTP view function
https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.0.md#responseObject
"""
status = int(status)
if not description:
description = HTTPStatus(status).description
def decorator(func: T) -> T:
if not hasattr(func, "__responses__"):
responses: typing.Dict[int, typing.Dict[str, typing.Any]] = {}
setattr(func, "__responses__", responses)
else:
responses = getattr(func, "__responses__")
responses[status] = {"description": description}
if content is not None:
responses[status]["content"] = content
if headers is not None:
responses[status]["headers"] = headers
if links is not None:
responses[status]["links"] = links
return func
return decorator
def normalize_raw_file_path(data: Dict[str, Any],
_: ContextType) -> Tuple[str, HTTPStatus]:
"""Cloud functions can be configured to trigger this function on any bucket that is being used as a test bed for
automatic uploads. This will just rename the incoming files to have a normalized path with a timestamp so
subsequent uploads do not have naming conflicts."""
project_id = os.environ.get(GCP_PROJECT_ID_KEY)
if not project_id:
error_str = (
"No project id set for call to direct ingest cloud function, returning."
)
logging.error(error_str)
return error_str, HTTPStatus.BAD_REQUEST
bucket = data["bucket"]
relative_file_path = data["name"]
url = _DIRECT_INGEST_NORMALIZE_RAW_PATH_URL.format(project_id, bucket,
relative_file_path)
logging.info("Calling URL: %s", url)
# Hit the cloud function backend, which will schedule jobs to parse
# data for unprocessed files in this bucket and persist to our database.
response = make_iap_request(url, IAP_CLIENT_ID[project_id])
logging.info("The response status is %s", response.status_code)
return "", HTTPStatus(response.status_code)
