def markdown(self, new_markdown: str) -> None: """Set the comment's markdown and render its HTML.""" if new_markdown == self.markdown: return self._markdown = new_markdown self.rendered_html = convert_markdown_to_safe_html(new_markdown) if (self.created_time and utc_now() - self.created_time > EDIT_GRACE_PERIOD): self.last_edited_time = utc_now()
def markdown(self, new_markdown: str) -> None: """Set the topic's markdown and render its HTML.""" if not self.is_text_type: raise AttributeError("Can only set markdown for text topics") if new_markdown == self.markdown: return self._markdown = new_markdown self.rendered_html = convert_markdown_to_safe_html(new_markdown) if self.created_time and utc_now() - self.created_time > EDIT_GRACE_PERIOD: self.last_edited_time = utc_now()
def close_voting_on_old_posts(config_path: str) -> None: """Update is_voting_closed column on all posts older than the voting period.""" db_session = get_session_from_config(config_path) db_session.query(Comment).filter( Comment.created_time < utc_now() - COMMENT_VOTING_PERIOD, Comment._is_voting_closed == False, # noqa ).update({"_is_voting_closed": True}, synchronize_session=False) db_session.query(Topic).filter( Topic.created_time < utc_now() - TOPIC_VOTING_PERIOD, Topic._is_voting_closed == False, # noqa ).update({"_is_voting_closed": True}, synchronize_session=False) db_session.commit()
def _increment_topic_comments_seen(request: Request, comment: Comment) -> None: """Increment the number of comments in a topic the user has viewed. If the user has the "track comment visits" feature enabled, we want to increment the number of comments they've seen in the thread that the comment came from, so that they don't *both* get a notification as well as have the thread highlight with "(1 new)". This should only happen if their last visit was before the comment was posted, however. Below, this is implemented as a INSERT ... ON CONFLICT DO UPDATE so that it will insert a new topic visit with 1 comment if they didn't previously have one at all. """ if request.user.track_comment_visits: statement = ( insert(TopicVisit.__table__) .values( user_id=request.user.user_id, topic_id=comment.topic_id, visit_time=utc_now(), num_comments=1, ) .on_conflict_do_update( constraint=TopicVisit.__table__.primary_key, set_={"num_comments": TopicVisit.num_comments + 1}, where=TopicVisit.visit_time < comment.created_time, ) ) request.db_session.execute(statement) mark_changed(request.db_session)
def create_topic(self) -> Topic: """Create and return an actual Topic for this scheduled topic.""" # if no user is specified, use the "generic"/automatic user (ID -1) if self.user: user = self.user else: user = (Session.object_session(self).query(User).filter( User.user_id == -1).one()) # treat both the title and markdown as Jinja templates (sandboxed) jinja_sandbox = SandboxedEnvironment() jinja_variables = {"current_time_utc": utc_now()} try: title_template = jinja_sandbox.from_string(self.title) title = title_template.render(jinja_variables) except: # pylint: disable=bare-except title = self.title try: markdown_template = jinja_sandbox.from_string(self.markdown) markdown = markdown_template.render(jinja_variables) except: # pylint: disable=bare-except markdown = self.markdown topic = Topic.create_text_topic(self.group, user, title, markdown) topic.tags = self.tags topic.schedule = self return topic
def advance_schedule_to_future(self) -> None: """Advance the schedule to the next future occurrence.""" if self.recurrence_rule: rule = self.recurrence_rule.replace(dtstart=self.next_post_time) self.next_post_time = rule.after(utc_now()) else: self.next_post_time = None
def edit(self, new_markdown: str, user: User, edit_message: str) -> None: """Set the page's markdown, render its HTML, and commit the repo.""" if new_markdown == self.markdown: return self.markdown = new_markdown self.rendered_html = convert_markdown_to_safe_html(new_markdown) self.rendered_html = add_anchors_to_headings(self.rendered_html) self.last_edited_time = utc_now() repo = Repository(self.BASE_PATH) author = Signature(user.username, user.username) repo.index.read() repo.index.add(str(self.file_path.relative_to(self.BASE_PATH))) repo.index.write() # Prepend the group name and page path to the edit message - if you change the # format of this, make sure to also change the page-editing template to match edit_message = f"~{self.group.path}/{self.path}: {edit_message}" repo.create_commit( repo.head.name, author, author, edit_message, repo.index.write_tree(), [repo.head.target], )
def mark_read_comment(request: Request) -> Response: """Mark a comment read (clear all notifications).""" comment = request.context request.query(CommentNotification).filter( CommentNotification.user == request.user, CommentNotification.comment == comment, ).update({CommentNotification.is_unread: False}, synchronize_session=False) # If the user has the "track comment visits" feature enabled, we want to # increment the number of comments they've seen in the thread that the # comment came from, so that they don't *both* get a notification as well # as have the thread highlight with "(1 new)". This should only happen if # their last visit was before the comment was posted, however. # Below, this is implemented as a INSERT ... ON CONFLICT DO UPDATE so that # it will insert a new topic visit with 1 comment if they didn't previously # have one at all. if request.user.track_comment_visits: statement = (insert(TopicVisit.__table__).values( user_id=request.user.user_id, topic_id=comment.topic_id, visit_time=utc_now(), num_comments=1, ).on_conflict_do_update( constraint=TopicVisit.__table__.primary_key, set_={'num_comments': TopicVisit.num_comments + 1}, where=TopicVisit.visit_time < comment.created_time, )) request.db_session.execute(statement) mark_changed(request.db_session) return IC_NOOP
def test_edit_after_grace_period(text_topic): """Ensure last_edited_time is set after the grace period.""" one_sec = timedelta(seconds=1) edit_time = text_topic.created_time + EDIT_GRACE_PERIOD + one_sec with freeze_time(edit_time): text_topic.markdown = "some new markdown" assert text_topic.last_edited_time == utc_now()
def markdown(self, new_markdown: str) -> None: """Set the comment's markdown and render its HTML.""" if new_markdown == self.markdown: return self._markdown = new_markdown self.rendered_html = convert_markdown_to_safe_html(new_markdown) extracted_text = extract_text_from_html( self.rendered_html, skip_tags=["blockquote"] ) self.excerpt = truncate_string( extracted_text, length=200, truncate_at_chars=" " ) if self.created_time and utc_now() - self.created_time > EDIT_GRACE_PERIOD: self.last_edited_time = utc_now()
def test_edit_after_grace_period(comment): """Ensure last_edited_time is set after the grace period.""" one_sec = timedelta(seconds=1) edit_time = comment.created_time + EDIT_GRACE_PERIOD + one_sec with freeze_time(edit_time): comment.markdown = 'some new markdown' assert comment.last_edited_time == utc_now()
def is_label_available(self, label: CommentLabelOption) -> bool: """Return whether the user has a particular label available.""" if label == CommentLabelOption.EXEMPLARY: if not self.last_exemplary_label_time: return True return utc_now() - self.last_exemplary_label_time > timedelta(hours=8) return True
def test_multiple_edits_update_time(comment): """Ensure multiple edits all update last_edited_time.""" one_sec = timedelta(seconds=1) initial_time = comment.created_time + EDIT_GRACE_PERIOD + one_sec for minutes in range(0, 4): edit_time = initial_time + timedelta(minutes=minutes) with freeze_time(edit_time): comment.markdown = f'edit #{minutes}' assert comment.last_edited_time == utc_now()
def test_multiple_edits_update_time(text_topic): """Ensure multiple edits all update last_edited_time.""" one_sec = timedelta(seconds=1) initial_time = text_topic.created_time + EDIT_GRACE_PERIOD + one_sec for minutes in range(0, 4): edit_time = initial_time + timedelta(minutes=minutes) with freeze_time(edit_time): text_topic.markdown = f"edit #{minutes}" assert text_topic.last_edited_time == utc_now()
def age(self) -> timedelta: """Return the model's age - requires it to have a `created_time` column.""" if not hasattr(self, "created_time"): raise AttributeError("'age' attribute requires 'created_time' column.") # created_time should only be None during __init__, age of 0 is reasonable if self.created_time is None: # type: ignore return timedelta(0) return utc_now() - self.created_time # type: ignore
def inside_time_period(self, period: SimpleHoursPeriod) -> "TopicQuery": """Restrict the topics to inside a time period (generative).""" # if the time period is too long, this will crash by creating a datetime outside # the valid range - catch that and just don't filter by time period at all if # the range is that large try: start_time = utc_now() - period.timedelta except OverflowError: return self return self.filter(Topic.created_time > start_time)
def _get_next_due_topic(db_session: Session) -> Optional[TopicSchedule]: """Get the next due topic (if any). Note that this also locks the topic's row with FOR UPDATE as well as using SKIP LOCKED. This should (hypothetically) mean that multiple instances of this script can run concurrently safely and will not attempt to post the same topics. """ return (db_session.query(TopicSchedule).filter( TopicSchedule.next_post_time <= utc_now()) # type: ignore .order_by(TopicSchedule.next_post_time).with_for_update( skip_locked=True).first())
def get_financials(request: Request) -> dict: """Display the financials page.""" financial_entries = (request.query(Financials).filter( Financials.date_range.op("@>")(text("CURRENT_DATE"))).order_by( Financials.entry_id).all()) # split the entries up by type entries: Dict[str, List] = defaultdict(list) for entry in financial_entries: entries[entry.entry_type.name.lower()].append(entry) return {"entries": entries, "current_time": utc_now()}
def process_message(self, message: Message) -> None: """Process a message from the stream.""" topic = ( self.db_session.query(Topic) .filter_by(topic_id=message.fields["topic_id"]) .one() ) if not topic.is_link_type: return if not self.scraper.is_applicable(topic.link): return # see if we already have a recent scrape result from the same url result = ( self.db_session.query(ScraperResult) .filter( ScraperResult.url == topic.link, ScraperResult.scraper_type == ScraperType.YOUTUBE, ScraperResult.scrape_time > utc_now() - RESCRAPE_DELAY, ) .order_by(desc(ScraperResult.scrape_time)) .first() ) # if not, scrape the url and store the result if not result: try: result = self.scraper.scrape_url(topic.link) except (HTTPError, ScraperError, Timeout): return self.db_session.add(result) new_metadata = YoutubeScraper.get_metadata_from_result(result) if new_metadata: # update the topic's content_metadata in a way that won't wipe out any # existing values, and can handle the column being null ( self.db_session.query(Topic) .filter(Topic.topic_id == topic.topic_id) .update( { "content_metadata": func.coalesce( Topic.content_metadata, cast({}, JSONB) ).op("||")(new_metadata) }, synchronize_session=False, ) )
def lift_expired_temporary_bans(config_path: str) -> None: """Lift temporary bans that have expired.""" db_session = get_session_from_config(config_path) db_session.query(User).filter( User.ban_expiry_time < utc_now(), # type: ignore User.is_banned == True, # noqa ).update({ "is_banned": False, "ban_expiry_time": None }, synchronize_session=False) db_session.commit()
def generate_insert_statement(cls, user: User, topic: Topic) -> Insert: """Return a INSERT ... ON CONFLICT DO UPDATE statement for a visit.""" visit_time = utc_now() return ( insert(cls.__table__) .values( user_id=user.user_id, topic_id=topic.topic_id, visit_time=visit_time, num_comments=topic.num_comments, ) .on_conflict_do_update( constraint=cls.__table__.primary_key, set_={"visit_time": visit_time, "num_comments": topic.num_comments}, ) )
def auth_principals(self) -> List[str]: """Return the user's authorization principals (used for permissions).""" principals: List[str] = [] # start with any principals manually defined in the permissions column if not self.permissions: pass elif isinstance(self.permissions, str): principals = [self.permissions] elif isinstance(self.permissions, list): principals = self.permissions else: raise ValueError("Unknown permissions format") # give the user the "comment.label" permission if they're over a week old if utc_now() - self.created_time > timedelta(days=7): principals.append("comment.label") return principals
def add_headers_to_response(self, response: Response) -> Response: """Add the relevant ratelimiting headers to a Response.""" # Retry-After: seconds the client should wait until retrying if self.time_until_retry: retry_seconds = int(self.time_until_retry.total_seconds()) response.headers["Retry-After"] = str(retry_seconds) # X-RateLimit-Limit: the total action limit (including used) response.headers["X-RateLimit-Limit"] = str(self.total_limit) # X-RateLimit-Remaining: remaining actions before client hits the limit response.headers["X-RateLimit-Remaining"] = str(self.remaining_limit) # X-RateLimit-Reset: epoch timestamp when limit will be back to full reset_time = utc_now() + self.time_until_max reset_timestamp = int(reset_time.timestamp()) response.headers["X-RateLimit-Reset"] = str(reset_timestamp) return response
def generate_stats(config_path: str) -> None: """Generate all stats for all groups for yesterday (UTC).""" db_session = get_session_from_config(config_path) # the end time is the start of the current day, start time 1 day before that end_time = utc_now().replace(hour=0, minute=0, second=0, microsecond=0) start_time = end_time - timedelta(days=1) groups = db_session.query(Group).all() for group in groups: with db_session.no_autoflush: db_session.add( topics_posted(db_session, group, start_time, end_time)) db_session.add( comments_posted(db_session, group, start_time, end_time)) try: db_session.commit() except IntegrityError: # stats have already run for this group/period combination, just skip continue
def get_settings_theme_previews(request: Request) -> dict: """Generate the theme preview page.""" # get the generic/unknown user and a random group to display on the example posts fake_user = request.query(User).filter(User.user_id == -1).one() group = request.query(Group).order_by(func.random()).limit(1).one() fake_link_topic = Topic.create_link_topic(group, fake_user, "Example Link Topic", "https://tildes.net/") fake_text_topic = Topic.create_text_topic(group, fake_user, "Example Text Topic", "No real text") fake_text_topic.content_metadata = { "excerpt": "Lorem ipsum dolor sit amet, consectetur adipiscing elit." } fake_topics = [fake_link_topic, fake_text_topic] # manually add other necessary attributes to the fake topics for fake_topic in fake_topics: fake_topic.topic_id = sys.maxsize fake_topic.tags = ["tag one", "tag two"] fake_topic.num_comments = 123 fake_topic.num_votes = 12 fake_topic.created_time = utc_now() - timedelta(hours=12) # create a fake top-level comment that appears to be written by the user markdown = ( "This is what a regular comment written by yourself would look like.\n\n" "It has **formatting** and a [link](https://tildes.net).") fake_top_comment = Comment(fake_link_topic, request.user, markdown) fake_top_comment.comment_id = sys.maxsize fake_top_comment.created_time = utc_now() - timedelta(hours=12, minutes=30) child_comments_markdown = [ ("This reply has received an Exemplary label. It also has a blockquote:\n\n" "> Hello World!"), ("This is a reply written by the topic's OP with a code block in it:\n\n" "```js\n" "function foo() {\n" " ['1', '2', '3'].map(parseInt);\n" "}\n" "```"), ("This reply is new and has the *Mark New Comments* stripe on its left " "(even if you don't have that feature enabled)."), ] fake_comments = [fake_top_comment] # vary the ID and created_time on each fake comment so CommentTree works properly current_comment_id = fake_top_comment.comment_id current_created_time = fake_top_comment.created_time for markdown in child_comments_markdown: current_comment_id -= 1 current_created_time += timedelta(minutes=5) fake_comment = Comment(fake_link_topic, fake_user, markdown, parent_comment=fake_top_comment) fake_comment.comment_id = current_comment_id fake_comment.created_time = current_created_time fake_comment.parent_comment_id = fake_top_comment.comment_id fake_comments.append(fake_comment) # add other necessary attributes to all of the fake comments for fake_comment in fake_comments: fake_comment.num_votes = 0 fake_tree = CommentTree(fake_comments, CommentTreeSortOption.NEWEST, request.user) # add a fake Exemplary label to the first child comment fake_comments[1].labels = [ CommentLabel(fake_comments[1], fake_user, CommentLabelOption.EXEMPLARY, 1.0) ] # the comment to mark as new is the last one, so set a visit time just before it fake_last_visit_time = fake_comments[-1].created_time - timedelta( minutes=1) return { "theme_options": THEME_OPTIONS, "fake_topics": fake_topics, "fake_comment_tree": fake_tree, "last_visit": fake_last_visit_time, }
def test_subsecond_descriptive_timedelta(): """Ensure time less than a second returns the special phrase.""" test_time = utc_now() - timedelta(microseconds=100) assert descriptive_timedelta(test_time) == 'a moment ago'
def get_group_topics( # noqa request: Request, after: Optional[str], before: Optional[str], order: Optional[TopicSortOption], per_page: int, rank_start: Optional[int], tag: Optional[Ltree], unfiltered: bool, **kwargs: Any) -> dict: """Get a listing of topics in the group.""" # period needs special treatment so we can distinguish between missing and None period = kwargs.get("period", missing) is_home_page = request.matched_route.name == "home" if is_home_page: # on the home page, include topics from the user's subscribed groups # (or all groups, if logged-out) if request.user: groups = [sub.group for sub in request.user.subscriptions] else: groups = [ group for group in request.query(Group).all() if group.path != "test" ] subgroups = None else: # otherwise, just topics from the single group that we're looking at groups = [request.context] subgroups = (request.query(Group).filter( Group.path.descendant_of(request.context.path), Group.path != request.context.path, ).all()) default_settings = _get_default_settings(request, order) if not order: order = default_settings.order if period is missing: period = default_settings.period # set up the basic query for topics query = (request.query(Topic).join_all_relationships().inside_groups( groups, include_subgroups=not is_home_page).exclude_ignored(). apply_sort_option(order)) # restrict the time period, if not set to "all time" if period: query = query.inside_time_period(period) # restrict to a specific tag, if we're viewing a single one if tag: query = query.has_tag(str(tag)) # apply before/after pagination restrictions if relevant if before: query = query.before_id36(before) if after: query = query.after_id36(after) # apply topic tag filters unless they're disabled or viewing a single tag if request.user and request.user.filtered_topic_tags and not (tag or unfiltered): query = query.filter(~Topic.tags.descendant_of( # type: ignore any_(cast(request.user.filtered_topic_tags, TagList)))) topics = query.get_page(per_page) # don't show pinned topics on home page if request.matched_route.name == "home": pinned_topics = [] else: # get pinned topics pinned_query = (request.query(Topic).join_all_relationships( ).inside_groups(groups).is_pinned(True).apply_sort_option(order)) pinned_topics = pinned_query.all() period_options = [ SimpleHoursPeriod(hours) for hours in (1, 12, 24, 72, 168) ] # add the current period to the bottom of the dropdown if it's not one of the # "standard" ones if period and period not in period_options: period_options.append(period) if isinstance(request.context, Group): wiki_pages = (request.query(GroupWikiPage).filter( GroupWikiPage.group == request.context).order_by( GroupWikiPage.path).all()) # remove the index from the page list, we'll output it separately if any(page.path == "index" for page in wiki_pages): wiki_has_index = True wiki_pages = [page for page in wiki_pages if page.path != "index"] else: wiki_has_index = False else: wiki_pages = None wiki_has_index = False if isinstance(request.context, Group): # Get the most recent topic from each scheduled topic in this group # I'm not even going to attempt to write this query in pure SQLAlchemy topic_id_subquery = """ SELECT topic_id FROM (SELECT topic_id, schedule_id, row_number() OVER (PARTITION BY schedule_id ORDER BY created_time DESC) AS rownum FROM topics) AS t WHERE schedule_id IS NOT NULL AND rownum = 1 """ most_recent_scheduled_topics = ( request.query(Topic).join(TopicSchedule).filter( Topic.topic_id.in_(text(topic_id_subquery)), # type: ignore TopicSchedule.group == request.context, TopicSchedule.next_post_time != None, # noqa ).order_by(TopicSchedule.next_post_time).all()) else: most_recent_scheduled_topics = None if is_home_page: financial_data = get_financial_data(request.db_session) else: financial_data = None return { "group": request.context, "groups": groups, "topics": topics, "pinned_topics": pinned_topics, "order": order, "order_options": TopicSortOption, "period": period, "period_options": period_options, "is_default_period": period == default_settings.period, "is_default_view": (period == default_settings.period and order == default_settings.order), "rank_start": rank_start, "tag": tag, "unfiltered": unfiltered, "wiki_pages": wiki_pages, "wiki_has_index": wiki_has_index, "subgroups": subgroups, "most_recent_scheduled_topics": most_recent_scheduled_topics, "financial_data": financial_data, "current_time": utc_now(), }
def test_utc_now_has_timezone(): """Ensure that utc_now() is returning a datetime with utc timezone.""" dt = utc_now() assert dt.tzinfo == timezone.utc
def test_above_second_descriptive_timedelta(): """Ensure it starts describing time in seconds above 1 second.""" test_time = utc_now() - timedelta(seconds=1, microseconds=100) assert descriptive_timedelta(test_time) == '1 second ago'
def inside_time_period(self, period: SimpleHoursPeriod) -> 'TopicQuery': """Restrict the topics to inside a time period (generative).""" return self.filter(Topic.created_time > utc_now() - period.timedelta)