def get_column_controls(
url: URL,
columns: typing.Dict[str, str],
selected_column: typing.Optional[str],
is_reverse: bool,
) -> typing.List[ColumnControl]:
controls = []
for column_id, name in columns.items():
if selected_column != column_id:
# Column is not selected. Link URL to forward search.
linked_url = url.include_query_params(
order=column_id).remove_query_params("page")
elif not is_reverse:
# Column is selected as a forward search. Link URL to reverse search.
linked_url = url.include_query_params(
order="-" + column_id).remove_query_params("page")
else:
# Column is selected as a reverse search. Link URL to remove search.
linked_url = url.remove_query_params("order").remove_query_params(
"page")
control = ColumnControl(
id=column_id,
text=name,
url=linked_url,
is_forward_sorted=selected_column == column_id and not is_reverse,
is_reverse_sorted=selected_column == column_id and is_reverse,
)
controls.append(control)
return controls
async def __call__(self, scope: Scope, receive: Receive,
send: Send) -> None:
scope["app"] = self
if scope["type"] in ("http",
"websocket"): # Handle some special routes
url = URL(scope=scope)
response: typing.Optional[Response] = None
# Force jump to https/wss
if Config().FORCE_SSL and scope["scheme"] in ("http", "ws"):
redirect_scheme = {"http": "https", "ws": "wss"}[url.scheme]
netloc = url.hostname if url.port in (80, 443) else url.netloc
url = url.replace(scheme=redirect_scheme, netloc=netloc)
response = RedirectResponse(
url, status_code=301) # for SEO, status code must be 301
if url.path == "/favicon.ico":
if os.path.exists(os.path.normpath("favicon.ico")):
response = FileResponse("favicon.ico")
if response is not None:
await response(scope, receive, send)
return
await self.asgiapp(scope, receive, send)
def test_url_from_scope():
u = URL(scope={
"path": "/path/to/somewhere",
"query_string": b"abc=123",
"headers": []
})
assert u == "/path/to/somewhere?abc=123"
assert repr(u) == "URL('/path/to/somewhere?abc=123')"
u = URL(
scope={
"scheme": "https",
"server": ("example.org", 123),
"path": "/path/to/somewhere",
"query_string": b"abc=123",
"headers": [],
})
assert u == "https://example.org:123/path/to/somewhere?abc=123"
assert repr(
u) == "URL('https://example.org:123/path/to/somewhere?abc=123')"
u = URL(
scope={
"scheme": "https",
"server": ("example.org", 443),
"path": "/path/to/somewhere",
"query_string": b"abc=123",
"headers": [],
})
assert u == "https://example.org/path/to/somewhere?abc=123"
assert repr(u) == "URL('https://example.org/path/to/somewhere?abc=123')"
async def test_upload(client, mock_csv):
user = await create_user()
client.login(user)
csv_file = open(mock_csv.name, "r")
url = app.url_path_for("profile", username=user["username"])
data = {"name": "new table"}
response = await client.post(url, data=data, allow_redirects=False)
expected_redirect = url
assert response.is_redirect
assert URL(response.headers["location"]).path == expected_redirect
url = app.url_path_for("upload",
username=user["username"],
table_id="new-table")
response = await client.post(url,
files={"upload-file": csv_file},
allow_redirects=False)
expected_redirect = app.url_path_for("table",
username=user["username"],
table_id="new-table")
assert response.is_redirect
assert URL(response.headers["location"]).path == expected_redirect
async def get_response(self, path: str, scope: Scope) -> Response:
"""
Returns an HTTP response, given the incoming path, method and request headers.
"""
if scope["method"] not in ("GET", "HEAD"):
return PlainTextResponse("Method Not Allowed", status_code=405)
full_path, stat_result = await self.lookup_path(path)
if stat_result and stat.S_ISREG(stat_result.st_mode):
# We have a static file to serve.
return self.file_response(full_path, stat_result, scope)
elif stat_result and stat.S_ISDIR(stat_result.st_mode) and self.html:
# We're in HTML mode, and have got a directory URL.
# Check if we have 'index.html' file to serve.
index_path = os.path.join(path, "index.html")
full_path, stat_result = await self.lookup_path(index_path)
if stat_result is not None and stat.S_ISREG(stat_result.st_mode):
if not scope["path"].endswith("/"):
# Directory URLs should redirect to always end in "/".
url = URL(scope=scope)
url = url.replace(path=url.path + "/")
return RedirectResponse(url=url)
return self.file_response(full_path, stat_result, scope)
if self.html:
# Check for '404.html' if we're in HTML mode.
full_path, stat_result = await self.lookup_path("404.html")
if stat_result is not None and stat.S_ISREG(stat_result.st_mode):
return self.file_response(
full_path, stat_result, scope, status_code=404
)
return PlainTextResponse("Not Found", status_code=404)
def test_get_column_controls_no_current_selection():
columns = {"username": "******", "email": "Email"}
url = URL("/")
column, is_reverse = None, False
controls = get_column_controls(url,
columns,
selected_column=None,
is_reverse=False)
assert controls == [
ColumnControl(
id="username",
text="Username",
url=URL("/?order=username"),
is_forward_sorted=False,
is_reverse_sorted=False,
),
ColumnControl(
id="email",
text="Email",
url=URL("/?order=email"),
is_forward_sorted=False,
is_reverse_sorted=False,
),
]
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if self.allow_any or scope["type"] not in (
"http",
"websocket",
): # pragma: no cover
await self.app(scope, receive, send)
return
headers = Headers(scope=scope)
host = headers.get("host", "").split(":")[0]
is_valid_host = False
found_www_redirect = False
for pattern in self.allowed_hosts:
if host == pattern or (
pattern.startswith("*") and host.endswith(pattern[1:])
):
is_valid_host = True
break
elif "www." + host == pattern:
found_www_redirect = True
if is_valid_host:
await self.app(scope, receive, send)
else:
response: Response
if found_www_redirect and self.www_redirect:
url = URL(scope=scope)
redirect_url = url.replace(netloc="www." + url.netloc)
response = RedirectResponse(url=str(redirect_url))
else:
response = PlainTextResponse("Invalid host header", status_code=400)
await response(scope, receive, send)
def test_get_response_headers():
dev_url = URL("https://dev.freeyourscience.org/index.html")
dev_headers = _get_response_headers(dev_url)
assert "X-Robots-Tag" in dev_headers
prod_url = URL("https://freeyourscience.org/index.html")
prod_headers = _get_response_headers(prod_url)
assert "X-Robots-Tag" not in prod_headers
def test_hidden_password():
u = URL("https://example.org/path/to/somewhere")
assert repr(u) == "URL('https://example.org/path/to/somewhere')"
u = URL("https://[email protected]/path/to/somewhere")
assert repr(u) == "URL('https://[email protected]/path/to/somewhere')"
u = URL("https://*****:*****@example.org/path/to/somewhere")
assert repr(u) == "URL('https://*****:*****@example.org/path/to/somewhere')"
def __call__(self, scope: Scope) -> ASGIInstance:
if scope["type"] in ("http",
"websocket") and scope["scheme"] in ("http",
"ws"):
redirect_scheme = {"http": "https", "ws": "wss"}[scope["scheme"]]
url = URL(scope=scope)
url = url.replace(scheme=redirect_scheme, netloc=url.hostname)
return RedirectResponse(url, status_code=301)
return self.app(scope)
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] in ("http", "websocket") and scope["scheme"] in ("http", "ws"):
url = URL(scope=scope)
redirect_scheme = {"http": "https", "ws": "wss"}[url.scheme]
netloc = url.hostname if url.port in (80, 443) else url.netloc
url = url.replace(scheme=redirect_scheme, netloc=netloc)
response = RedirectResponse(url, status_code=307)
await response(scope, receive, send)
else:
await self.app(scope, receive, send)
def test_url():
u = URL("https://example.org:123/path/to/somewhere?abc=123#anchor")
assert u.scheme == "https"
assert u.hostname == "example.org"
assert u.port == 123
assert u.netloc == "example.org:123"
assert u.username is None
assert u.password is None
assert u.path == "/path/to/somewhere"
assert u.query == "abc=123"
assert u.params == ""
assert u.fragment == "anchor"
new = u.replace(scheme="http")
assert new == "http://example.org:123/path/to/somewhere?abc=123#anchor"
assert new.scheme == "http"
def __call__(self, scope: Scope) -> ASGIInstance:
if scope["type"] in ("http"):
host = None
for key, value in scope["headers"]:
if key == b"host":
host = value.decode("latin-1")
break
if host and not host.startswith("www."):
url = URL(scope=scope)
url = url.replace(hostname=f"www.{host}")
return RedirectResponse(url, status_code=301)
return self.app(scope)
async def check_recaptcha(request: Request,
recaptcha_token: Optional[str],
*,
error_headers=None) -> None:
client_ip = get_ip(request)
if not recaptcha_token:
logger.warning('recaptcha token not provided, path="%s" ip=%s',
request.url.path, client_ip)
raise exceptions.HttpBadRequest('No recaptcha value',
headers=error_headers)
settings: BaseSettings = glove.settings
post_data = {
'secret': settings.recaptcha_secret,
'response': recaptcha_token,
'remoteip': client_ip
}
r = await glove.http.post(settings.recaptcha_url, data=post_data)
r.raise_for_status()
data = r.json()
# if settings.origin exists, use that instead of the request to avoid problems with old browsers that don't include
# the Origin header
if origin := getattr(settings, 'origin', None):
request_host = URL(origin).hostname
async def test_valid_edit(client):
"""
Test row edit.
"""
user = await create_user()
table, columns, rows = await create_table(user)
client.login(user)
url = app.url_path_for(
"detail",
username=user["username"],
table_id=table["identity"],
row_uuid=rows[0]["uuid"],
)
data = {
"constituency": "Harrow East",
"surname": "WALLACE",
"first_name": "Emma",
"party": "Green Party",
"votes": 846,
}
response = await client.post(url, data=data, allow_redirects=False)
expected_redirect = url
assert response.is_redirect
assert URL(response.headers["location"]).path == expected_redirect
def test_single_page_does_not_include_any_pagination_controls():
"""
When there is only a single page, no pagination controls should render.
"""
url = URL("/")
controls = get_page_controls(url, current_page=1, total_pages=1)
assert controls == []
def __call__(self, scope: Scope) -> ASGIInstance:
assert scope["type"] in ("http", "websocket", "lifespan")
if scope["type"] == "lifespan":
return LifespanHandler(scope)
if "router" not in scope:
scope["router"] = self
partial = None
for route in self.routes:
match, child_scope = route.matches(scope)
if match == Match.FULL:
scope.update(child_scope)
return route(scope)
elif match == Match.PARTIAL and partial is None:
partial = route
partial_scope = child_scope
if partial is not None:
scope.update(partial_scope)
return partial(scope)
if self.redirect_slashes and not scope["path"].endswith("/"):
redirect_scope = dict(scope)
redirect_scope["path"] += "/"
for route in self.routes:
match, child_scope = route.matches(redirect_scope)
if match != Match.NONE:
redirect_url = URL(scope=redirect_scope)
return RedirectResponse(url=str(redirect_url))
return self.default(scope)
class ProdSettings(BaseConfig):
""" Configuration class for site production environment """
config = Config()
DEBUG = False
DB_USER = config("DB_USER", cast=str)
DB_PASSWORD = config("DB_PASSWORD", cast=Secret)
DB_HOST = config("DB_HOST", cast=str)
DB_PORT = config("DB_PORT", cast=str)
DB_NAME = config("DB_NAME", cast=str)
INCLUDE_SCHEMA=config("INCLUDE_SCHEMA", cast=bool)
SSL_CERT_FILE = config("SSL_CERT_FILE")
SQLALCHEMY_ECHO = config("SQLALCHEMY_ECHO",cast=bool,default=False)
DATABASE_URL = config(
"DATABASE_URL",
cast=make_url,
default=URL(
drivername="asyncpg",
username=DB_USER,
password=DB_PASSWORD,
host=DB_HOST,
port=DB_PORT,
database=DB_NAME,
),
)
async def __call__(self, scope: Scope, receive: Receive,
send: Send) -> None:
headers = Headers(scope=scope)
url = URL(scope=scope)
log_debug(message=f'[API Server] {url}',
data={'headers': f'{headers}'})
await self.app(scope, receive, send)
def get_place(
id: str,
request: Request,
background_tasks: BackgroundTasks,
lang: str = None,
type: Optional[PlaceType] = Query(
None, description="Restrict the type of documents to search in."
),
verbosity: Verbosity = Verbosity.default(),
) -> Place:
"""Main handler that returns the requested place."""
lang = validate_lang(lang)
try:
place = place_from_id(id, type)
except InvalidPlaceId as e:
raise HTTPException(status_code=404, detail=e.message) from e
except PlaceNotFound as e:
raise HTTPException(status_code=404, detail=e.message) from e
except RedirectToPlaceId as e:
path_prefix = request.headers.get("x-forwarded-prefix", "").rstrip("/")
path = request.app.url_path_for("get_place", id=e.target_id)
query = request.url.query
return JSONResponse(
status_code=303,
headers={"location": str(URL(path=f"{path_prefix}{path}", query=query))},
content={"id": e.target_id},
)
log_place_request(place, request.headers)
if settings["BLOCK_COVID_ENABLED"] and settings["COVID19_USE_REDIS_DATASET"]:
background_tasks.add_task(covid19_osm_task)
return place.load_place(lang, verbosity)
def base_url(self) -> URL:
base_url_scope = dict(self.scope)
base_url_scope["path"] = "/"
base_url_scope["query_string"] = b""
base_url_scope["root_path"] = base_url_scope.get(
"app_root_path", base_url_scope.get("root_path", ""))
return URL(scope=base_url_scope)
async def __call__(self, scope: Scope, receive: Receive, send: Send):
scope["send"] = send # See: `RequestResponseMiddleware`.
if scope["type"] == "lifespan":
await self.lifespan(scope, receive, send)
return
route = self._find_route(scope)
if route is not None:
try:
await route(scope, receive, send)
return
except Redirect as exc:
scope["res"] = exc.response
return
try_http_redirect = (scope["type"] == "http"
and not scope["path"].endswith("/")
and redirect_trailing_slash_enabled())
if try_http_redirect:
redirect_scope = dict(scope)
redirect_scope["path"] += "/"
route = self._find_route(redirect_scope)
if route is not None:
redirect_url = URL(scope=redirect_scope)
scope["res"] = Redirect(str(redirect_url)).response
return
if scope["type"] == "websocket":
await WebSocketClose(code=403)(receive, send)
return
raise HTTPError(404)
def is_valid_referer(request: Request) -> bool:
header: str = request.headers.get("origin") or request.headers.get("referer") or ""
url = URL(header)
hostname = url.hostname
is_valid = hostname in settings.ALLOWED_HOSTS if hostname else False
return is_valid
def __init__(self, request: HTTPConnection):
super().__init__(
request.scope["method"],
str(URL(scope=request.scope)),
None,
Headers(scope=request.scope),
)
self._orig_request = request
async def __call__(self, scope: Scope, receive: Receive,
send: Send) -> None:
handler = getattr(self, scope["type"])
url = URL(scope=scope)
if url.path.endswith("/index"):
handler = RedirectResponse(
url.replace(path=url.path[:-len("/index")]),
status_code=301,
)
elif "_" in url.path and not self.allow_underline:
handler = RedirectResponse(
url.replace(path=url.path.replace("_", "-")),
status_code=301,
)
await handler(scope, receive, send)
def url_path_for(self, name: str, **path_params: str) -> URL:
for route in self.routes or []:
try:
url = route.url_path_for(name, **path_params)
return URL(scheme=url.scheme, path=self.path + url.path)
except NoMatchFound as exc:
pass
raise NoMatchFound()
def is_valid_referer(self, request: Request) -> bool:
header: str = (request.headers.get("origin")
or request.headers.get("referer") or "")
url = URL(header)
hostname = url.hostname
is_valid = hostname in self.allowed_hosts if hostname else False
return is_valid
async def get_response(self, path: str, scope: Scope) -> Response:
"""
Returns an HTTP response, given the incoming path, method and request headers.
"""
if scope["method"] not in ("GET", "HEAD"):
raise HTTPException(status_code=405)
try:
full_path, stat_result = await anyio.to_thread.run_sync(
self.lookup_path, path)
except PermissionError:
raise HTTPException(status_code=401)
except OSError:
raise
if stat_result and stat.S_ISREG(stat_result.st_mode):
# We have a static file to serve.
return self.file_response(full_path, stat_result, scope)
elif stat_result and stat.S_ISDIR(stat_result.st_mode) and self.html:
# We're in HTML mode, and have got a directory URL.
# Check if we have 'index.html' file to serve.
index_path = os.path.join(path, "index.html")
full_path, stat_result = await anyio.to_thread.run_sync(
self.lookup_path, index_path)
if stat_result is not None and stat.S_ISREG(stat_result.st_mode):
if not scope["path"].endswith("/"):
# Directory URLs should redirect to always end in "/".
url = URL(scope=scope)
url = url.replace(path=url.path + "/")
return RedirectResponse(url=url)
return self.file_response(full_path, stat_result, scope)
if self.html:
# Check for '404.html' if we're in HTML mode.
full_path, stat_result = await anyio.to_thread.run_sync(
self.lookup_path, "404.html")
if stat_result and stat.S_ISREG(stat_result.st_mode):
return FileResponse(
full_path,
stat_result=stat_result,
method=scope["method"],
status_code=404,
)
raise HTTPException(status_code=404)
def base_url(self) -> URL:
if not hasattr(self, "_base_url"):
base_url_scope = dict(self.scope)
base_url_scope["path"] = "/"
base_url_scope["query_string"] = b""
base_url_scope["root_path"] = base_url_scope.get(
"app_root_path", base_url_scope.get("root_path", ""))
self._base_url = URL(scope=base_url_scope)
return self._base_url
async def get_upstream_raw_response(request: Request):
config = app.extra['config']
async with httpx.AsyncClient(http_versions=['HTTP/1.1']) as client:
upstream_url = URL(config.upstream.url)
url = request.url.replace(netloc=upstream_url.netloc, path=(upstream_url.path.rstrip('/')+request.url.path))
url = xURL(str(url))
body = await request.body()
return await client.request(
request.method, url, data=body, headers=request.headers.raw, cookies=request.cookies, allow_redirects=False)
