def __init__(
self,
queue_name: str,
routing_keys: Sequence[str],
uses_db: bool = True,
) -> None:
"""Initialize a new queue, bindings, and consumer for it."""
self.connection = Connection()
self.channel = self.connection.channel()
self.channel.queue_declare(queue_name, durable=True, auto_delete=False)
for routing_key in routing_keys:
self.channel.queue_bind(
queue_name,
exchange=self.PGSQL_EXCHANGE_NAME,
routing_key=routing_key,
)
if uses_db:
self.db_session = get_session_from_config(os.environ['INI_FILE'])
else:
self.db_session = None
super().__init__(self.channel, queue_name)
def update_common_topic_tags(config_path: str) -> None:
"""Update the list of common topic tags for all groups."""
db_session = get_session_from_config(config_path)
all_groups = db_session.query(Group).all()
for group in all_groups:
# create a subquery for all tags from topics in that group - UNNEST() converts
# the arrays of tags into rows so that we can easily group and count, and
# created_time will be used to determine when a particular tag was last used
group_tags = (db_session.query(
func.unnest(Topic.tags).label("tag"),
Topic.created_time).filter(Topic.group == group).subquery())
# get the list of the most common tags, based on frequency and breaking ties
# with which was used most recently
common_tags = (db_session.query(
group_tags.columns["tag"],
func.count().label("frequency"),
func.max(group_tags.columns["created_time"]).label("last_used"),
).group_by("tag").order_by(
desc("frequency"),
desc("last_used")).limit(MAX_NUM_COMMON_TAGS).all())
group.common_topic_tags = [common_tag[0] for common_tag in common_tags]
db_session.add(group)
db_session.commit()
def clean_all_data(config_path: str) -> None:
"""Clean all private/deleted data.
This should generally be the only function called in most cases, and will initiate
the full cleanup process.
"""
db_session = get_session_from_config(config_path)
cleaner = DataCleaner(db_session, RETENTION_PERIOD)
cleaner.clean_all()
def insert_dev_data(config_path: str) -> None:
"""Load the app config and insert some "starter" data for a dev version."""
session = get_session_from_config(config_path)
user = User("TestUser", "password")
group = Group("testing", "An automatically created group to use for testing")
subscription = GroupSubscription(user, group)
session.add_all([user, group, subscription])
session.commit()
def post_scheduled_topics(config_path: str) -> None:
"""Post all scheduled topics that are due to be posted."""
db_session = get_session_from_config(config_path)
due_topic = _get_next_due_topic(db_session)
while due_topic:
db_session.add(due_topic.create_topic())
due_topic.advance_schedule_to_future()
db_session.add(due_topic)
db_session.commit()
due_topic = _get_next_due_topic(db_session)
def insert_dev_data(config_path: str) -> None:
"""Load the app config and insert some "starter" data for a dev version."""
session = get_session_from_config(config_path)
session.add_all([
User('TestUser', 'password'),
Group(
'testing',
'An automatically created group to use for testing purposes',
),
])
session.commit()
def lift_expired_temporary_bans(config_path: str) -> None:
"""Lift temporary bans that have expired."""
db_session = get_session_from_config(config_path)
db_session.query(User).filter(
User.ban_expiry_time < utc_now(), # type: ignore
User.is_banned == True, # noqa
).update({
"is_banned": False,
"ban_expiry_time": None
},
synchronize_session=False)
db_session.commit()
def close_voting_on_old_posts(config_path: str) -> None:
"""Update is_voting_closed column on all posts older than the voting period."""
db_session = get_session_from_config(config_path)
db_session.query(Comment).filter(
Comment.created_time < utc_now() - COMMENT_VOTING_PERIOD,
Comment._is_voting_closed == False, # noqa
).update({"_is_voting_closed": True}, synchronize_session=False)
db_session.query(Topic).filter(
Topic.created_time < utc_now() - TOPIC_VOTING_PERIOD,
Topic._is_voting_closed == False, # noqa
).update({"_is_voting_closed": True}, synchronize_session=False)
db_session.commit()
def initialize_db(config_path: str, alembic_config_path: Optional[str] = None) -> None:
"""Load the app config and create the database tables."""
db_session = get_session_from_config(config_path)
engine = db_session.bind
create_tables(engine)
run_sql_scripts_in_dir("sql/init/", engine)
# if an Alembic config file wasn't specified, assume it's alembic.ini in the same
# directory
if not alembic_config_path:
path = os.path.split(config_path)[0]
alembic_config_path = os.path.join(path, "alembic.ini")
# mark current Alembic revision in db so migrations start from this point
alembic_cfg = Config(alembic_config_path)
command.stamp(alembic_cfg, "head")
def __init__(
self,
consumer_group: str,
source_streams: Sequence[str],
uses_db: bool = True,
skip_pending: bool = False,
):
"""Initialize a new consumer, creating consumer groups and streams if needed."""
ini_file_path = os.environ["INI_FILE"]
config = ConfigParser()
config.read(ini_file_path)
self.redis = Redis(
unix_socket_path=config.get("app:main", "redis.unix_socket_path"))
self.consumer_group = consumer_group
self.source_streams = [
f"{REDIS_KEY_PREFIX}{stream}" for stream in source_streams
]
# hardcoded for now, will need to change for multiple consumers in same group
self.name = f"{consumer_group}-1"
# create all the consumer groups and streams (if necessary)
for stream in self.source_streams:
try:
self.redis.xgroup_create(stream, consumer_group, mkstream=True)
except ResponseError as error:
# if the consumer group already exists, a BUSYGROUP error will be
# returned, so we want to ignore that one but raise anything else
if not str(error).startswith("BUSYGROUP"):
raise
if uses_db:
self.db_session = get_session_from_config(ini_file_path)
else:
self.db_session = None
# start by reading any already-pending messages by default
self.is_reading_pending = not skip_pending
self._init_metrics()
def generate_stats(config_path: str) -> None:
"""Generate all stats for all groups for yesterday (UTC)."""
db_session = get_session_from_config(config_path)
# the end time is the start of the current day, start time 1 day before that
end_time = utc_now().replace(hour=0, minute=0, second=0, microsecond=0)
start_time = end_time - timedelta(days=1)
groups = db_session.query(Group).all()
for group in groups:
with db_session.no_autoflush:
db_session.add(
topics_posted(db_session, group, start_time, end_time))
db_session.add(
comments_posted(db_session, group, start_time, end_time))
try:
db_session.commit()
except IntegrityError:
# stats have already run for this group/period combination, just skip
continue
def initialize_db(config_path: str,
alembic_config_path: Optional[str] = None) -> None:
"""Load the app config and create the database tables.
This function will probably only complete successfully when run as user postgres,
since the run_sql_scripts_in_dir method runs psql through that user to be able to
create functions using untrusted languages (PL/Python specifically).
"""
db_session = get_session_from_config(config_path)
engine = db_session.bind
create_tables(engine)
run_sql_scripts_in_dir("sql/init/", engine)
# if an Alembic config file wasn't specified, assume it's alembic.ini in the same
# directory
if not alembic_config_path:
path = os.path.split(config_path)[0]
alembic_config_path = os.path.join(path, "alembic.ini")
# mark current Alembic revision in db so migrations start from this point
alembic_cfg = Config(alembic_config_path)
command.stamp(alembic_cfg, "head")