def markdown(self, new_markdown: str) -> None:
"""Set the comment's markdown and render its HTML."""
if new_markdown == self.markdown:
return
self._markdown = new_markdown
self.rendered_html = convert_markdown_to_safe_html(new_markdown)
if (self.created_time
and utc_now() - self.created_time > EDIT_GRACE_PERIOD):
self.last_edited_time = utc_now()
def markdown(self, new_markdown: str) -> None:
"""Set the topic's markdown and render its HTML."""
if not self.is_text_type:
raise AttributeError("Can only set markdown for text topics")
if new_markdown == self.markdown:
return
self._markdown = new_markdown
self.rendered_html = convert_markdown_to_safe_html(new_markdown)
if self.created_time and utc_now() - self.created_time > EDIT_GRACE_PERIOD:
self.last_edited_time = utc_now()
def close_voting_on_old_posts(config_path: str) -> None:
"""Update is_voting_closed column on all posts older than the voting period."""
db_session = get_session_from_config(config_path)
db_session.query(Comment).filter(
Comment.created_time < utc_now() - COMMENT_VOTING_PERIOD,
Comment._is_voting_closed == False, # noqa
).update({"_is_voting_closed": True}, synchronize_session=False)
db_session.query(Topic).filter(
Topic.created_time < utc_now() - TOPIC_VOTING_PERIOD,
Topic._is_voting_closed == False, # noqa
).update({"_is_voting_closed": True}, synchronize_session=False)
db_session.commit()
def _increment_topic_comments_seen(request: Request, comment: Comment) -> None:
"""Increment the number of comments in a topic the user has viewed.
If the user has the "track comment visits" feature enabled, we want to increment the
number of comments they've seen in the thread that the comment came from, so that
they don't *both* get a notification as well as have the thread highlight with "(1
new)". This should only happen if their last visit was before the comment was
posted, however. Below, this is implemented as a INSERT ... ON CONFLICT DO UPDATE
so that it will insert a new topic visit with 1 comment if they didn't previously
have one at all.
"""
if request.user.track_comment_visits:
statement = (
insert(TopicVisit.__table__)
.values(
user_id=request.user.user_id,
topic_id=comment.topic_id,
visit_time=utc_now(),
num_comments=1,
)
.on_conflict_do_update(
constraint=TopicVisit.__table__.primary_key,
set_={"num_comments": TopicVisit.num_comments + 1},
where=TopicVisit.visit_time < comment.created_time,
)
)
request.db_session.execute(statement)
mark_changed(request.db_session)
def create_topic(self) -> Topic:
"""Create and return an actual Topic for this scheduled topic."""
# if no user is specified, use the "generic"/automatic user (ID -1)
if self.user:
user = self.user
else:
user = (Session.object_session(self).query(User).filter(
User.user_id == -1).one())
# treat both the title and markdown as Jinja templates (sandboxed)
jinja_sandbox = SandboxedEnvironment()
jinja_variables = {"current_time_utc": utc_now()}
try:
title_template = jinja_sandbox.from_string(self.title)
title = title_template.render(jinja_variables)
except: # pylint: disable=bare-except
title = self.title
try:
markdown_template = jinja_sandbox.from_string(self.markdown)
markdown = markdown_template.render(jinja_variables)
except: # pylint: disable=bare-except
markdown = self.markdown
topic = Topic.create_text_topic(self.group, user, title, markdown)
topic.tags = self.tags
topic.schedule = self
return topic
def advance_schedule_to_future(self) -> None:
"""Advance the schedule to the next future occurrence."""
if self.recurrence_rule:
rule = self.recurrence_rule.replace(dtstart=self.next_post_time)
self.next_post_time = rule.after(utc_now())
else:
self.next_post_time = None
def edit(self, new_markdown: str, user: User, edit_message: str) -> None:
"""Set the page's markdown, render its HTML, and commit the repo."""
if new_markdown == self.markdown:
return
self.markdown = new_markdown
self.rendered_html = convert_markdown_to_safe_html(new_markdown)
self.rendered_html = add_anchors_to_headings(self.rendered_html)
self.last_edited_time = utc_now()
repo = Repository(self.BASE_PATH)
author = Signature(user.username, user.username)
repo.index.read()
repo.index.add(str(self.file_path.relative_to(self.BASE_PATH)))
repo.index.write()
# Prepend the group name and page path to the edit message - if you change the
# format of this, make sure to also change the page-editing template to match
edit_message = f"~{self.group.path}/{self.path}: {edit_message}"
repo.create_commit(
repo.head.name,
author,
author,
edit_message,
repo.index.write_tree(),
[repo.head.target],
)
def mark_read_comment(request: Request) -> Response:
"""Mark a comment read (clear all notifications)."""
comment = request.context
request.query(CommentNotification).filter(
CommentNotification.user == request.user,
CommentNotification.comment == comment,
).update({CommentNotification.is_unread: False}, synchronize_session=False)
# If the user has the "track comment visits" feature enabled, we want to
# increment the number of comments they've seen in the thread that the
# comment came from, so that they don't *both* get a notification as well
# as have the thread highlight with "(1 new)". This should only happen if
# their last visit was before the comment was posted, however.
# Below, this is implemented as a INSERT ... ON CONFLICT DO UPDATE so that
# it will insert a new topic visit with 1 comment if they didn't previously
# have one at all.
if request.user.track_comment_visits:
statement = (insert(TopicVisit.__table__).values(
user_id=request.user.user_id,
topic_id=comment.topic_id,
visit_time=utc_now(),
num_comments=1,
).on_conflict_do_update(
constraint=TopicVisit.__table__.primary_key,
set_={'num_comments': TopicVisit.num_comments + 1},
where=TopicVisit.visit_time < comment.created_time,
))
request.db_session.execute(statement)
mark_changed(request.db_session)
return IC_NOOP
def test_edit_after_grace_period(text_topic):
"""Ensure last_edited_time is set after the grace period."""
one_sec = timedelta(seconds=1)
edit_time = text_topic.created_time + EDIT_GRACE_PERIOD + one_sec
with freeze_time(edit_time):
text_topic.markdown = "some new markdown"
assert text_topic.last_edited_time == utc_now()
def markdown(self, new_markdown: str) -> None:
"""Set the comment's markdown and render its HTML."""
if new_markdown == self.markdown:
return
self._markdown = new_markdown
self.rendered_html = convert_markdown_to_safe_html(new_markdown)
extracted_text = extract_text_from_html(
self.rendered_html, skip_tags=["blockquote"]
)
self.excerpt = truncate_string(
extracted_text, length=200, truncate_at_chars=" "
)
if self.created_time and utc_now() - self.created_time > EDIT_GRACE_PERIOD:
self.last_edited_time = utc_now()
def test_edit_after_grace_period(comment):
"""Ensure last_edited_time is set after the grace period."""
one_sec = timedelta(seconds=1)
edit_time = comment.created_time + EDIT_GRACE_PERIOD + one_sec
with freeze_time(edit_time):
comment.markdown = 'some new markdown'
assert comment.last_edited_time == utc_now()
def is_label_available(self, label: CommentLabelOption) -> bool:
"""Return whether the user has a particular label available."""
if label == CommentLabelOption.EXEMPLARY:
if not self.last_exemplary_label_time:
return True
return utc_now() - self.last_exemplary_label_time > timedelta(hours=8)
return True
def test_multiple_edits_update_time(comment):
"""Ensure multiple edits all update last_edited_time."""
one_sec = timedelta(seconds=1)
initial_time = comment.created_time + EDIT_GRACE_PERIOD + one_sec
for minutes in range(0, 4):
edit_time = initial_time + timedelta(minutes=minutes)
with freeze_time(edit_time):
comment.markdown = f'edit #{minutes}'
assert comment.last_edited_time == utc_now()
def test_multiple_edits_update_time(text_topic):
"""Ensure multiple edits all update last_edited_time."""
one_sec = timedelta(seconds=1)
initial_time = text_topic.created_time + EDIT_GRACE_PERIOD + one_sec
for minutes in range(0, 4):
edit_time = initial_time + timedelta(minutes=minutes)
with freeze_time(edit_time):
text_topic.markdown = f"edit #{minutes}"
assert text_topic.last_edited_time == utc_now()
def age(self) -> timedelta:
"""Return the model's age - requires it to have a `created_time` column."""
if not hasattr(self, "created_time"):
raise AttributeError("'age' attribute requires 'created_time' column.")
# created_time should only be None during __init__, age of 0 is reasonable
if self.created_time is None: # type: ignore
return timedelta(0)
return utc_now() - self.created_time # type: ignore
def inside_time_period(self, period: SimpleHoursPeriod) -> "TopicQuery":
"""Restrict the topics to inside a time period (generative)."""
# if the time period is too long, this will crash by creating a datetime outside
# the valid range - catch that and just don't filter by time period at all if
# the range is that large
try:
start_time = utc_now() - period.timedelta
except OverflowError:
return self
return self.filter(Topic.created_time > start_time)
def _get_next_due_topic(db_session: Session) -> Optional[TopicSchedule]:
"""Get the next due topic (if any).
Note that this also locks the topic's row with FOR UPDATE as well as using SKIP
LOCKED. This should (hypothetically) mean that multiple instances of this script
can run concurrently safely and will not attempt to post the same topics.
"""
return (db_session.query(TopicSchedule).filter(
TopicSchedule.next_post_time <= utc_now()) # type: ignore
.order_by(TopicSchedule.next_post_time).with_for_update(
skip_locked=True).first())
def get_financials(request: Request) -> dict:
"""Display the financials page."""
financial_entries = (request.query(Financials).filter(
Financials.date_range.op("@>")(text("CURRENT_DATE"))).order_by(
Financials.entry_id).all())
# split the entries up by type
entries: Dict[str, List] = defaultdict(list)
for entry in financial_entries:
entries[entry.entry_type.name.lower()].append(entry)
return {"entries": entries, "current_time": utc_now()}
def process_message(self, message: Message) -> None:
"""Process a message from the stream."""
topic = (
self.db_session.query(Topic)
.filter_by(topic_id=message.fields["topic_id"])
.one()
)
if not topic.is_link_type:
return
if not self.scraper.is_applicable(topic.link):
return
# see if we already have a recent scrape result from the same url
result = (
self.db_session.query(ScraperResult)
.filter(
ScraperResult.url == topic.link,
ScraperResult.scraper_type == ScraperType.YOUTUBE,
ScraperResult.scrape_time > utc_now() - RESCRAPE_DELAY,
)
.order_by(desc(ScraperResult.scrape_time))
.first()
)
# if not, scrape the url and store the result
if not result:
try:
result = self.scraper.scrape_url(topic.link)
except (HTTPError, ScraperError, Timeout):
return
self.db_session.add(result)
new_metadata = YoutubeScraper.get_metadata_from_result(result)
if new_metadata:
# update the topic's content_metadata in a way that won't wipe out any
# existing values, and can handle the column being null
(
self.db_session.query(Topic)
.filter(Topic.topic_id == topic.topic_id)
.update(
{
"content_metadata": func.coalesce(
Topic.content_metadata, cast({}, JSONB)
).op("||")(new_metadata)
},
synchronize_session=False,
)
)
def lift_expired_temporary_bans(config_path: str) -> None:
"""Lift temporary bans that have expired."""
db_session = get_session_from_config(config_path)
db_session.query(User).filter(
User.ban_expiry_time < utc_now(), # type: ignore
User.is_banned == True, # noqa
).update({
"is_banned": False,
"ban_expiry_time": None
},
synchronize_session=False)
db_session.commit()
def generate_insert_statement(cls, user: User, topic: Topic) -> Insert:
"""Return a INSERT ... ON CONFLICT DO UPDATE statement for a visit."""
visit_time = utc_now()
return (
insert(cls.__table__)
.values(
user_id=user.user_id,
topic_id=topic.topic_id,
visit_time=visit_time,
num_comments=topic.num_comments,
)
.on_conflict_do_update(
constraint=cls.__table__.primary_key,
set_={"visit_time": visit_time, "num_comments": topic.num_comments},
)
)
def auth_principals(self) -> List[str]:
"""Return the user's authorization principals (used for permissions)."""
principals: List[str] = []
# start with any principals manually defined in the permissions column
if not self.permissions:
pass
elif isinstance(self.permissions, str):
principals = [self.permissions]
elif isinstance(self.permissions, list):
principals = self.permissions
else:
raise ValueError("Unknown permissions format")
# give the user the "comment.label" permission if they're over a week old
if utc_now() - self.created_time > timedelta(days=7):
principals.append("comment.label")
return principals
def add_headers_to_response(self, response: Response) -> Response:
"""Add the relevant ratelimiting headers to a Response."""
# Retry-After: seconds the client should wait until retrying
if self.time_until_retry:
retry_seconds = int(self.time_until_retry.total_seconds())
response.headers["Retry-After"] = str(retry_seconds)
# X-RateLimit-Limit: the total action limit (including used)
response.headers["X-RateLimit-Limit"] = str(self.total_limit)
# X-RateLimit-Remaining: remaining actions before client hits the limit
response.headers["X-RateLimit-Remaining"] = str(self.remaining_limit)
# X-RateLimit-Reset: epoch timestamp when limit will be back to full
reset_time = utc_now() + self.time_until_max
reset_timestamp = int(reset_time.timestamp())
response.headers["X-RateLimit-Reset"] = str(reset_timestamp)
return response
def generate_stats(config_path: str) -> None:
"""Generate all stats for all groups for yesterday (UTC)."""
db_session = get_session_from_config(config_path)
# the end time is the start of the current day, start time 1 day before that
end_time = utc_now().replace(hour=0, minute=0, second=0, microsecond=0)
start_time = end_time - timedelta(days=1)
groups = db_session.query(Group).all()
for group in groups:
with db_session.no_autoflush:
db_session.add(
topics_posted(db_session, group, start_time, end_time))
db_session.add(
comments_posted(db_session, group, start_time, end_time))
try:
db_session.commit()
except IntegrityError:
# stats have already run for this group/period combination, just skip
continue
def get_settings_theme_previews(request: Request) -> dict:
"""Generate the theme preview page."""
# get the generic/unknown user and a random group to display on the example posts
fake_user = request.query(User).filter(User.user_id == -1).one()
group = request.query(Group).order_by(func.random()).limit(1).one()
fake_link_topic = Topic.create_link_topic(group, fake_user,
"Example Link Topic",
"https://tildes.net/")
fake_text_topic = Topic.create_text_topic(group, fake_user,
"Example Text Topic",
"No real text")
fake_text_topic.content_metadata = {
"excerpt": "Lorem ipsum dolor sit amet, consectetur adipiscing elit."
}
fake_topics = [fake_link_topic, fake_text_topic]
# manually add other necessary attributes to the fake topics
for fake_topic in fake_topics:
fake_topic.topic_id = sys.maxsize
fake_topic.tags = ["tag one", "tag two"]
fake_topic.num_comments = 123
fake_topic.num_votes = 12
fake_topic.created_time = utc_now() - timedelta(hours=12)
# create a fake top-level comment that appears to be written by the user
markdown = (
"This is what a regular comment written by yourself would look like.\n\n"
"It has **formatting** and a [link](https://tildes.net).")
fake_top_comment = Comment(fake_link_topic, request.user, markdown)
fake_top_comment.comment_id = sys.maxsize
fake_top_comment.created_time = utc_now() - timedelta(hours=12, minutes=30)
child_comments_markdown = [
("This reply has received an Exemplary label. It also has a blockquote:\n\n"
"> Hello World!"),
("This is a reply written by the topic's OP with a code block in it:\n\n"
"```js\n"
"function foo() {\n"
" ['1', '2', '3'].map(parseInt);\n"
"}\n"
"```"),
("This reply is new and has the *Mark New Comments* stripe on its left "
"(even if you don't have that feature enabled)."),
]
fake_comments = [fake_top_comment]
# vary the ID and created_time on each fake comment so CommentTree works properly
current_comment_id = fake_top_comment.comment_id
current_created_time = fake_top_comment.created_time
for markdown in child_comments_markdown:
current_comment_id -= 1
current_created_time += timedelta(minutes=5)
fake_comment = Comment(fake_link_topic,
fake_user,
markdown,
parent_comment=fake_top_comment)
fake_comment.comment_id = current_comment_id
fake_comment.created_time = current_created_time
fake_comment.parent_comment_id = fake_top_comment.comment_id
fake_comments.append(fake_comment)
# add other necessary attributes to all of the fake comments
for fake_comment in fake_comments:
fake_comment.num_votes = 0
fake_tree = CommentTree(fake_comments, CommentTreeSortOption.NEWEST,
request.user)
# add a fake Exemplary label to the first child comment
fake_comments[1].labels = [
CommentLabel(fake_comments[1], fake_user, CommentLabelOption.EXEMPLARY,
1.0)
]
# the comment to mark as new is the last one, so set a visit time just before it
fake_last_visit_time = fake_comments[-1].created_time - timedelta(
minutes=1)
return {
"theme_options": THEME_OPTIONS,
"fake_topics": fake_topics,
"fake_comment_tree": fake_tree,
"last_visit": fake_last_visit_time,
}
def test_subsecond_descriptive_timedelta():
"""Ensure time less than a second returns the special phrase."""
test_time = utc_now() - timedelta(microseconds=100)
assert descriptive_timedelta(test_time) == 'a moment ago'
def get_group_topics( # noqa
request: Request, after: Optional[str], before: Optional[str],
order: Optional[TopicSortOption], per_page: int,
rank_start: Optional[int], tag: Optional[Ltree], unfiltered: bool,
**kwargs: Any) -> dict:
"""Get a listing of topics in the group."""
# period needs special treatment so we can distinguish between missing and None
period = kwargs.get("period", missing)
is_home_page = request.matched_route.name == "home"
if is_home_page:
# on the home page, include topics from the user's subscribed groups
# (or all groups, if logged-out)
if request.user:
groups = [sub.group for sub in request.user.subscriptions]
else:
groups = [
group for group in request.query(Group).all()
if group.path != "test"
]
subgroups = None
else:
# otherwise, just topics from the single group that we're looking at
groups = [request.context]
subgroups = (request.query(Group).filter(
Group.path.descendant_of(request.context.path),
Group.path != request.context.path,
).all())
default_settings = _get_default_settings(request, order)
if not order:
order = default_settings.order
if period is missing:
period = default_settings.period
# set up the basic query for topics
query = (request.query(Topic).join_all_relationships().inside_groups(
groups, include_subgroups=not is_home_page).exclude_ignored().
apply_sort_option(order))
# restrict the time period, if not set to "all time"
if period:
query = query.inside_time_period(period)
# restrict to a specific tag, if we're viewing a single one
if tag:
query = query.has_tag(str(tag))
# apply before/after pagination restrictions if relevant
if before:
query = query.before_id36(before)
if after:
query = query.after_id36(after)
# apply topic tag filters unless they're disabled or viewing a single tag
if request.user and request.user.filtered_topic_tags and not (tag or
unfiltered):
query = query.filter(~Topic.tags.descendant_of( # type: ignore
any_(cast(request.user.filtered_topic_tags, TagList))))
topics = query.get_page(per_page)
# don't show pinned topics on home page
if request.matched_route.name == "home":
pinned_topics = []
else:
# get pinned topics
pinned_query = (request.query(Topic).join_all_relationships(
).inside_groups(groups).is_pinned(True).apply_sort_option(order))
pinned_topics = pinned_query.all()
period_options = [
SimpleHoursPeriod(hours) for hours in (1, 12, 24, 72, 168)
]
# add the current period to the bottom of the dropdown if it's not one of the
# "standard" ones
if period and period not in period_options:
period_options.append(period)
if isinstance(request.context, Group):
wiki_pages = (request.query(GroupWikiPage).filter(
GroupWikiPage.group == request.context).order_by(
GroupWikiPage.path).all())
# remove the index from the page list, we'll output it separately
if any(page.path == "index" for page in wiki_pages):
wiki_has_index = True
wiki_pages = [page for page in wiki_pages if page.path != "index"]
else:
wiki_has_index = False
else:
wiki_pages = None
wiki_has_index = False
if isinstance(request.context, Group):
# Get the most recent topic from each scheduled topic in this group
# I'm not even going to attempt to write this query in pure SQLAlchemy
topic_id_subquery = """
SELECT topic_id FROM (SELECT topic_id, schedule_id, row_number() OVER
(PARTITION BY schedule_id ORDER BY created_time DESC) AS rownum FROM topics)
AS t WHERE schedule_id IS NOT NULL AND rownum = 1
"""
most_recent_scheduled_topics = (
request.query(Topic).join(TopicSchedule).filter(
Topic.topic_id.in_(text(topic_id_subquery)), # type: ignore
TopicSchedule.group == request.context,
TopicSchedule.next_post_time != None, # noqa
).order_by(TopicSchedule.next_post_time).all())
else:
most_recent_scheduled_topics = None
if is_home_page:
financial_data = get_financial_data(request.db_session)
else:
financial_data = None
return {
"group":
request.context,
"groups":
groups,
"topics":
topics,
"pinned_topics":
pinned_topics,
"order":
order,
"order_options":
TopicSortOption,
"period":
period,
"period_options":
period_options,
"is_default_period":
period == default_settings.period,
"is_default_view": (period == default_settings.period
and order == default_settings.order),
"rank_start":
rank_start,
"tag":
tag,
"unfiltered":
unfiltered,
"wiki_pages":
wiki_pages,
"wiki_has_index":
wiki_has_index,
"subgroups":
subgroups,
"most_recent_scheduled_topics":
most_recent_scheduled_topics,
"financial_data":
financial_data,
"current_time":
utc_now(),
}
def test_utc_now_has_timezone():
"""Ensure that utc_now() is returning a datetime with utc timezone."""
dt = utc_now()
assert dt.tzinfo == timezone.utc
def test_above_second_descriptive_timedelta():
"""Ensure it starts describing time in seconds above 1 second."""
test_time = utc_now() - timedelta(seconds=1, microseconds=100)
assert descriptive_timedelta(test_time) == '1 second ago'
def inside_time_period(self, period: SimpleHoursPeriod) -> 'TopicQuery':
"""Restrict the topics to inside a time period (generative)."""
return self.filter(Topic.created_time > utc_now() - period.timedelta)