def pipeline():
redis = redis_connection()
pipeline = redis.pipeline()
pipeline.delete("ratelimit-ws:127.0.0.1")
pipeline.delete("ratelimit:127.0.0.1")
pipeline.execute()
yield pipeline
def rediss():
""" Create a RedisStub. """
def mock_get(section, key):
return "none"
with mock.patch("aurweb.config.get", side_effect=mock_get):
aurweb.config.rehash()
redis = redis_connection()
aurweb.config.rehash()
yield redis
def pipeline():
redis = redis_connection()
pipeline = redis.pipeline()
# The 'testclient' host is used when requesting the app
# via fastapi.testclient.TestClient.
pipeline.delete("ratelimit-ws:testclient")
pipeline.delete("ratelimit:testclient")
pipeline.execute()
yield pipeline
def redis():
redis = redis_connection()
def delete_keys():
# Cleanup keys if they exist.
for key in ("package_count", "orphan_count", "user_count",
"trusted_user_count", "seven_days_old_added",
"seven_days_old_updated", "year_old_updated",
"never_updated", "package_updates"):
if redis.get(key) is not None:
redis.delete(key)
delete_keys()
yield redis
delete_keys()
def updated_packages(limit: int = 0,
cache_ttl: int = 600) -> List[models.Package]:
""" Return a list of valid Package objects ordered by their
ModifiedTS column in descending order from cache, after setting
the cache when no key yet exists.
:param limit: Optional record limit
:param cache_ttl: Cache expiration time (in seconds)
:return: A list of Packages
"""
redis = redis_connection()
packages = redis.get("package_updates")
if packages:
# If we already have a cache, deserialize it and return.
return orjson.loads(packages)
with db.begin():
query = db.query(models.Package).join(models.PackageBase).filter(
models.PackageBase.PackagerUID.isnot(None)).order_by(
models.PackageBase.ModifiedTS.desc())
if limit:
query = query.limit(limit)
packages = []
for pkg in query:
# For each Package returned by the query, append a dict
# containing Package columns we're interested in.
packages.append({
"Name": pkg.Name,
"Version": pkg.Version,
"PackageBase": {
"ModifiedTS": pkg.PackageBase.ModifiedTS
}
})
# Store the JSON serialization of the package_updates key into Redis.
redis.set("package_updates", orjson.dumps(packages))
redis.expire("package_updates", cache_ttl)
# Return the deserialized list of packages.
return packages
async def internal_server_error(request: Request, exc: Exception) -> Response:
"""
Catch all uncaught Exceptions thrown in a route.
:param request: FastAPI Request
:return: Rendered 500.html template with status_code 500
"""
repo = aurweb.config.get("notifications", "gitlab-instance")
project = aurweb.config.get("notifications", "error-project")
token = aurweb.config.get("notifications", "error-token")
context = make_context(request, "Internal Server Error")
# Print out the exception via `traceback` and store the value
# into the `traceback` context variable.
tb_io = io.StringIO()
traceback.print_exc(file=tb_io)
tb = tb_io.getvalue()
context["traceback"] = tb
# Produce a SHA1 hash of the traceback string.
tb_hash = hashlib.sha1(tb.encode()).hexdigest()
tb_id = tb_hash[:7]
redis = redis_connection()
key = f"tb:{tb_hash}"
retval = redis.get(key)
if not retval:
# Expire in one hour; this is just done to make sure we
# don't infinitely store these values, but reduce the number
# of automated reports (notification below). At this time of
# writing, unexpected exceptions are not common, thus this
# will not produce a large memory footprint in redis.
pipe = redis.pipeline()
pipe.set(key, tb)
pipe.expire(key, 86400) # One day.
pipe.execute()
# Send out notification about it.
if "set-me" not in (project, token):
proj = quote_plus(project)
endp = f"{repo}/api/v4/projects/{proj}/issues"
base = f"{request.url.scheme}://{request.url.netloc}"
title = f"Traceback [{tb_id}]: {base}{request.url.path}"
desc = [
"DISCLAIMER",
"----------",
"**This issue is confidential** and should be sanitized "
"before sharing with users or developers. Please ensure "
"you've completed the following tasks:",
"- [ ] I have removed any sensitive data and "
"the description history.",
"",
"Exception Details",
"-----------------",
f"- Route: `{request.url.path}`",
f"- User: `{request.user.Username}`",
f"- Email: `{request.user.Email}`",
]
# Add method-specific information to the description.
if request.method.lower() == "get":
# get
if request.url.query:
desc = desc + [f"- Query: `{request.url.query}`"]
desc += ["", f"```{tb}```"]
else:
# post
form_data = str(dict(request.state.form_data))
desc = desc + [f"- Data: `{form_data}`"] + ["", f"```{tb}```"]
headers = {"Authorization": f"Bearer {token}"}
data = {
"title": title,
"description": "\n".join(desc),
"labels": ["triage"],
"confidential": True,
}
logger.info(endp)
resp = requests.post(endp, json=data, headers=headers)
if resp.status_code != http.HTTPStatus.CREATED:
logger.error(
f"Unable to report exception to {repo}: {resp.text}")
else:
logger.warning("Unable to report an exception found due to "
"unset notifications.error-{{project,token}}")
# Log details about the exception traceback.
logger.error(f"FATAL[{tb_id}]: An unexpected exception has occurred.")
logger.error(tb)
else:
retval = retval.decode()
return render_template(request,
"errors/500.html",
context,
status_code=http.HTTPStatus.INTERNAL_SERVER_ERROR)