def reindex_bookmarks(log_level: str):
"""Requests an (re)index of the most recent crawl for each bookmark."""
configure_logging(log_level)
log.warning("requesting reindex of all bookmarks")
Session = get_session_cls()
index = 0
with contextlib.closing(Session()) as session:
for index, crawl_uuid in enumerate(
most_recent_successful_bookmark_crawls(session)):
publish_message(IndexRequested(crawl_uuid),
environ["QM_RABBITMQ_BG_WORKER_TOPIC"])
log.warning("requested %d indexings", index)
def get_crawl_body(crawl_uuid: UUID, log_level: str):
configure_logging(log_level)
Session = get_session_cls()
with contextlib.closing(Session()) as session:
metadata = get_crawl_metadata(session, crawl_uuid)
bucket = file_storage.get_response_body_bucket()
filelike = file_storage.download_file(bucket, str(metadata.body_uuid))
while True:
buf = filelike.read(io.DEFAULT_BUFFER_SIZE)
if len(buf) == 0:
break
stdout.write(buf.decode("utf-8"))
def bg_worker(log_level):
load_config()
proc.set_dlq(
missive.dlq.sqlite.SQLiteDLQ(
environ["QM_MISSIVE_SQLITE_DLQ_CONNSTRING"]))
configure_logging(log_level)
adapted_proc = RabbitMQAdapter(
PickleMessage,
proc,
[environ["QM_RABBITMQ_BG_WORKER_TOPIC"]],
url_or_conn=environ["QM_RABBITMQ_URL"],
)
adapted_proc.run()
def init_app() -> flask.Flask:
configure_logging()
load_config(env_ini=environ.get("QM_ENV_INI", None))
app = flask.Flask("quarchive")
app.config["SECRET_KEY"] = environ["QM_SECRET_KEY"]
app.config["SQLALCHEMY_DATABASE_URI"] = environ["QM_SQL_URL"]
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
app.config["CRYPT_CONTEXT"] = CryptContext(["argon2", "bcrypt"])
app.config["SESSION_COOKIE_SAMESITE"] = "Lax"
# This configuration option, enabled by default, causes cookies to be
# refreshed (echoed) for many endpoints where that's not appropriate - eg
# those in the icon blueprint
app.config["SESSION_REFRESH_EACH_REQUEST"] = False
# By default Postgres will consult the locale to decide what timezone to
# return datetimes in. We want UTC in all cases.
app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {
"connect_args": {
"options": "-c timezone=utc"
}
}
app.config["PAGE_SIZE"] = 30
db.init_app(app)
with app.app_context():
log.info("using engine: %s", db.session.bind)
cors.init_app(app)
babel = Babel(app,
default_locale="en_GB",
default_timezone="Europe/London")
app.register_blueprint(web_blueprint)
app.register_blueprint(icon_blueprint)
app.register_blueprint(sync_blueprint)
@babel.timezoneselector
def use_user_timezone():
current_user = get_current_user()
if current_user is not None:
tzinfo = current_user.timezone
log.debug("using user timezone: %s", tzinfo)
return tzinfo.zone
@app.context_processor
def urlsplit_cp():
return {"urlunsplit": urlunsplit}
@app.template_global(name="tag_colour")
def tag_colour_template_function(tag: str) -> int:
return tag_colour(tag)
@app.template_global(name="modify_query")
def modify_query(**new_args):
args = flask.request.args.copy()
for key, value in new_args.items():
if value is not None:
args[key] = value
else:
# None is a request to unset
del args[key]
return "?%s" % url_encode(args)
@app.before_first_request
def log_db() -> None:
flask.current_app.logger.info("using engine: %s", db.session.bind)
return app
def quarchive_cli(log_level) -> None:
configure_logging(log_level)