Exemple #1
0
    def _mock_comment(id=1,
                      author_id=1,
                      link_id=1,
                      sr_id=1,
                      can_comment=True,
                      can_view_promo=True,
                      is_moderator=False,
                      **kwargs):
        kwargs['id'] = id
        kwargs['author_id'] = author_id
        kwargs['link_id'] = link_id

        comment = Comment(**kwargs)
        VByName.run = MagicMock(return_value=comment)

        link = Link(id=link_id)
        Link._byID = MagicMock(return_value=link)

        sr = Subreddit(id=sr_id)
        comment.subreddit_slow = sr
        link.subreddit_slow = sr

        Subreddit.can_comment = MagicMock(return_value=can_comment)
        Link.can_view_promo = MagicMock(return_value=can_view_promo)
        Subreddit.is_moderator = MagicMock(return_value=is_moderator)

        return comment
Exemple #2
0
    def _run_new_comment(msg):
        fname = msg.body
        comment = Comment._by_fullname(fname, data=True)
        sr = Subreddit._byID(comment.sr_id)

        add_queries(
            [get_all_comments(), get_sr_comments(sr)], insert_items=[comment])
Exemple #3
0
def process_comments_on_post(post, comments):
    for comment in comments:
        if comment_exists(post, comment):
            continue

        # Prepare data for import
        ip = '127.0.0.1'
        naive_date = datetime.datetime.strptime(comment['dateCreated'], DATE_FORMAT)
        local_date = INPUT_TIMEZONE.localize(naive_date, is_dst=False) # Pick the non daylight savings time
        utc_date = local_date.astimezone(pytz.utc)

        # Determine account to use for this comment
        account = get_or_create_account(comment['author'])

        if not dryrun:
            # Create new comment
            new_comment, inbox_rel = Comment._new(account, post, None, comment['body'], ip, date=utc_date)
            new_comment.is_html = True
            new_comment.ob_imported = True
            new_comment._commit()

        try:
            print " Imported as '%s' %s" % (account.name.decode('utf-8').encode('utf-8'), comment_excerpt(comment).decode('utf-8').encode('utf-8'))
        except UnicodeError:
            print " Imported comment"
Exemple #4
0
    def process_comment(self, comment_data, comment, post):
        # Prepare data for import
        ip = '127.0.0.1'
        if comment_data:
            naive_date = datetime.datetime.strptime(comment_data['dateCreated'], DATE_FORMAT)
            local_date = INPUT_TIMEZONE.localize(naive_date, is_dst=False) # Pick the non daylight savings time
            utc_date = local_date.astimezone(pytz.utc)

            # Determine account to use for this comment
            account = self._get_or_create_account(comment_data['author'], comment_data['authorEmail'])

        if comment_data and not comment:
            # Create new comment
            comment, inbox_rel = Comment._new(account, post, None, comment_data['body'], ip, date=utc_date)
            comment.is_html = True
            comment.ob_imported = True
            comment._commit()
        elif comment_data and comment:
            # Overwrite existing comment
            comment.author_id = account._id
            comment.body = comment_data['body']
            comment.ip = ip
            comment._date = utc_date
            comment.is_html = True
            comment.ob_imported = True
            comment._commit()
        elif not comment_data and comment:
            # Not enough comment data being imported to overwrite all comments
            print 'WARNING: More comments in lesswrong than we are importing, ignoring additional comment in lesswrong'
Exemple #5
0
    def process_message(msg):
        from r2.lib.db.queries import (
            add_queries,
            add_to_commentstree_q,
            get_comments,
        )

        vote_data = json.loads(msg.body)
        hook = hooks.get_hook('vote.validate_vote_data')
        if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
            # Corrupt records in the queue. Ignore them.
            print "Ignoring invalid vote by %s on %s %s" % (
                vote_data.get('user_id', '<unknown>'),
                vote_data.get('thing_fullname', '<unknown>'), vote_data)
            return

        timer = g.stats.get_timer("comment_vote_processor")
        timer.start()

        user = Account._byID(vote_data.pop("user_id"))
        comment = Comment._by_fullname(vote_data.pop("thing_fullname"))

        print "Processing vote by %s on %s %s" % (user, comment, vote_data)

        try:
            vote = Vote(
                user,
                comment,
                direction=vote_data["direction"],
                date=datetime.utcfromtimestamp(vote_data["date"]),
                data=vote_data["data"],
                event_data=vote_data.get("event_data"),
            )
        except TypeError as e:
            # a vote on an invalid type got in the queue, just skip it
            g.log.exception("Invalid type: %r", e.message)
            return

        vote.commit()
        timer.intermediate("create_vote_object")

        vote_valid = vote.is_automatic_initial_vote or vote.effects.affects_score
        comment_valid = not (comment._spam or comment._deleted)
        if vote_valid and comment_valid:
            author = Account._byID(comment.author_id)
            add_queries(
                queries=[get_comments(author, sort, 'all') for sort in SORTS],
                insert_items=comment,
            )
            timer.intermediate("author_queries")

            # update the score periodically when a comment has many votes
            update_threshold = g.live_config['comment_vote_update_threshold']
            update_period = g.live_config['comment_vote_update_period']
            num_votes = comment.num_votes
            if num_votes <= update_threshold or num_votes % update_period == 0:
                add_to_commentstree_q(comment)

        timer.stop()
        timer.flush()
Exemple #6
0
def activate_names_requested_in(link):
    tree = get_comment_tree(link)
    acceptable_names = []
    if tree.tree:
        top_level_cids = tree.tree[None]
        comments = chain.from_iterable(Comment._byID(chunk, return_dict=False,
                                                     data=True)
                                       for chunk in in_chunks(top_level_cids))

        for comment in sorted(comments, key=lambda c: c._ups, reverse=True):
            if comment._spam or comment._deleted:
                continue

            sanitized = comment.body.strip()
            match = valid_name_re.search(sanitized)
            if match:
                acceptable_names.append((comment, match.group(1)))

    # we activate one name for each 100% of rev goal met
    names = acceptable_names[:link.revenue_bucket]
    activate_names(link, names)

    activated_names = [name for comment, name in names]
    link.server_names = activated_names
    link.flair_text = ", ".join(activated_names) if names else "/dev/null"
    link.flair_css_class = "goal-bucket-%d" % link.revenue_bucket
    link._commit()
Exemple #7
0
    def get_all_comments(self):
        from r2.lib.db import queries
        from r2.models import Comment
        from r2.controllers.errors import UserRequiredException

        if not c.user_is_loggedin:
            raise UserRequiredException

        friends = self.get_important_friends(c.user._id)

        if not friends:
            return []

        if g.use_query_cache:
            # with the precomputer enabled, this Subreddit only supports
            # being sorted by 'new'. it would be nice to have a
            # cleaner UI than just blatantly ignoring their sort,
            # though
            sort = 'new'
            time = 'all'

            friends = Account._byID(friends, return_dict=False)

            crs = [
                queries.get_comments(friend, sort, time) for friend in friends
            ]
            return queries.MergedCachedResults(crs)

        else:
            q = Comment._query(Comment.c.author_id == friends,
                               sort=desc('_date'),
                               data=True)
            return q
Exemple #8
0
    def _run_commentstree(msgs, chan):
        comments = Comment._by_fullname([msg.body for msg in msgs],
                                        data=True,
                                        return_dict=False)
        print 'Processing %r' % (comments, )

        add_comment_tree(comments)
Exemple #9
0
def get_deleted_comments(user_id):
    return Comment._query(
        Comment.c.author_id == user_id,
        Comment.c._deleted == True,
        Comment.c._spam == (True, False),
        sort=db_sort("new"),
    )
Exemple #10
0
    def _mock_comment(self,
                      id=1,
                      author_id=1,
                      link_id=1,
                      sr_id=1,
                      can_comment=True,
                      can_view_promo=True,
                      is_moderator=False,
                      **kwargs):
        kwargs['id'] = id
        kwargs['author_id'] = author_id
        kwargs['link_id'] = link_id
        kwargs['sr_id'] = sr_id

        comment = Comment(**kwargs)
        self.autopatch(VByName, "run", return_value=comment)

        link = Link(id=link_id, sr_id=sr_id)
        self.autopatch(Link, "_byID", return_value=link)

        sr = Subreddit(id=sr_id)
        self.autopatch(Subreddit, "_byID", return_value=sr)
        self.autopatch(Subreddit, "can_comment", return_value=can_comment)
        self.autopatch(Link, "can_view_promo", return_value=can_view_promo)
        self.autopatch(Subreddit, "is_moderator", return_value=is_moderator)

        return comment
Exemple #11
0
    def process_comment(self, comment_data, comment, post, comment_dictionary):
        # Prepare data for import
        ip = '127.0.0.1'
        if comment_data:
            naive_date = datetime.datetime.strptime(comment_data['dateCreated'], DATE_FORMAT)
            local_date = INPUT_TIMEZONE.localize(naive_date, is_dst=False) # Pick the non daylight savings time
            utc_date = local_date.astimezone(pytz.utc)

            # Determine account to use for this comment
            account = self._get_or_create_account(comment_data['author'], comment_data['authorEmail'])

        if comment_data and not comment_data['author'].endswith("| The Effective Altruism Blog"):
            if not comment:
                # Create new comment
                comment, inbox_rel = Comment._new(account, post, None, comment_data['body'], ip, date=utc_date)
                if str(comment_data['commentParent']) in comment_dictionary:
                    comment.parent_id = comment_dictionary[str(comment_data['commentParent'])]
                comment.is_html = True
                comment.ob_imported = True
                comment._commit()
                comment_dictionary[str(comment_data['commentId'])] = comment._id
            else:
                # Overwrite existing comment
                if str(comment_data['commentParent']) in comment_dictionary:
                    comment.parent_id = comment_dictionary[str(comment_data['commentParent'])]
                comment.author_id = account._id
                comment.body = comment_data['body']
                comment.ip = ip
                comment._date = utc_date
                comment.is_html = True
                comment.ob_imported = True
                comment._commit()
                comment_dictionary[str(comment_data['commentId'])] = comment._id
Exemple #12
0
    def rollback_account_votes(self, account):
        query = LinkVotesByAccount._cf.xget(account._id36)
        for thing_id, vote_state in query:
            link = Link._byID36(thing_id)
            if int(vote_state) == int(
                    Vote.SERIALIZED_DIRECTIONS[Vote.DIRECTIONS.onon]):  # 3
                link._incr("_ups", -1)
                link._incr("_downs", -1)
            elif int(vote_state) == int(
                    Vote.SERIALIZED_DIRECTIONS[Vote.DIRECTIONS.onoff]):  # 4
                link._incr("_ups", -1)
            elif int(vote_state) == int(
                    Vote.SERIALIZED_DIRECTIONS[Vote.DIRECTIONS.offon]):  # 5
                link._incr("_downs", -1)

        query = CommentVotesByAccount._cf.xget(account._id36)
        for thing_id, vote_state in query:
            comment = Comment._byID36(thing_id)
            if int(vote_state) == int(
                    Vote.SERIALIZED_DIRECTIONS[Vote.DIRECTIONS.onon]):  # 3
                comment._incr("_ups", -1)
                comment._incr("_downs", -1)
            elif int(vote_state) == int(
                    Vote.SERIALIZED_DIRECTIONS[Vote.DIRECTIONS.onoff]):  # 4
                comment._incr("_ups", -1)
            elif int(vote_state) == int(
                    Vote.SERIALIZED_DIRECTIONS[Vote.DIRECTIONS.offon]):  # 5
                comment._incr("_downs", -1)
Exemple #13
0
    def get_all_comments(self):
        from r2.lib.db import queries
        from r2.models import Comment
        from r2.controllers.errors import UserRequiredException

        if not c.user_is_loggedin:
            raise UserRequiredException

        friends = self.get_important_friends(c.user._id)

        if not friends:
            return []

        if g.use_query_cache:
            # with the precomputer enabled, this Subreddit only supports
            # being sorted by 'new'. it would be nice to have a
            # cleaner UI than just blatantly ignoring their sort,
            # though
            sort = "new"
            time = "all"

            friends = Account._byID(friends, return_dict=False)

            crs = [queries.get_comments(friend, sort, time) for friend in friends]
            return queries.MergedCachedResults(crs)

        else:
            q = Comment._query(Comment.c.author_id == friends, sort=desc("_date"), data=True)
            return q
Exemple #14
0
    def _run_new_comment(msg):
        fname = msg.body
        comment = Comment._by_fullname(fname,data=True)
        sr = Subreddit._byID(comment.sr_id)

        add_queries([get_all_comments(),
                     get_sr_comments(sr)],
                    insert_items = [comment])
Exemple #15
0
    def _mock_comment(id=1, author_id=1, link_id=1, sr_id=1, **kwargs):
        kwargs['id'] = id
        kwargs['author_id'] = author_id
        kwargs['link_id'] = link_id

        comment = Comment(**kwargs)
        VByName.run = MagicMock(return_value=comment)

        link = Link(id=link_id)
        Link._byID = MagicMock(return_value=link)

        sr = Subreddit(id=sr_id)
        comment.subreddit_slow = MagicMock(return_value=sr)
        comment.subreddit_slow.is_moderator = MagicMock(return_value=False)
        link.subreddit = sr

        return comment
Exemple #16
0
    def _run_new_comments(msgs, chan):
        fnames = [msg.body for msg in msgs]

        comments = Comment._by_fullname(fnames, data=True, return_dict=False)
        add_queries([get_all_comments()], insert_items=comments)

        bysrid = _by_srid(comments, False)
        for srid, sr_comments in bysrid.iteritems():
            add_queries([_get_sr_comments(srid)], insert_items=sr_comments)
Exemple #17
0
    def _run_new_comments(msgs, chan):
        fnames = [msg.body for msg in msgs]

        comments = Comment._by_fullname(fnames, data=True, return_dict=False)
        add_queries([get_all_comments()], insert_items=comments)

        bysrid = _by_srid(comments, False)
        for srid, sr_comments in bysrid.iteritems():
            add_queries([_get_sr_comments(srid)], insert_items=sr_comments)
Exemple #18
0
    def post_process_post(self, post):
        """Perform post processsing to rewrite URLs and generate mapping
           between old and new permalinks"""
        post.article = self.rewrite_ob_urls(post.article)
        post._commit()

        comments = Comment._query(Comment.c.link_id == post._id, data=True)
        for comment in comments:
            comment.body = self.rewrite_ob_urls(comment.body)
            comment._commit()
Exemple #19
0
 def post_process_post(self, post):
     """Perform post processsing to rewrite URLs and generate mapping
        between old and new permalinks"""
     post.article = self.rewrite_ob_urls(post.article)
     post._commit()
     
     comments = Comment._query(Comment.c.link_id == post._id, data = True)
     for comment in comments:
         comment.body = self.rewrite_ob_urls(comment.body)
         comment._commit()
Exemple #20
0
def comment_reply_effect(comment):
    if comment.parent_id is not None:
        parent = Comment._byID(comment.parent_id, data=True)
    else:
        parent = Link._byID(comment.link_id, data=True)
    all_effects = effects.get_all_effects([parent._fullname])
    parent_effects = all_effects.get(parent._fullname, [])
    for item_name in parent_effects:
        item = items.get_item(item_name)
        item.on_reply(c.user, parent)
Exemple #21
0
def comment_reply_effect(comment):
    if comment.parent_id is not None:
        parent = Comment._byID(comment.parent_id, data=True)
    else:
        parent = Link._byID(comment.link_id, data=True)
    all_effects = effects.get_all_effects([parent._fullname])
    parent_effects = all_effects.get(parent._fullname, [])
    for item_name in parent_effects:
        item = items.get_item(item_name)
        item.on_reply(c.user, parent)
Exemple #22
0
def _populate(after_id=None, estimate=54301242):
    from r2.models import Comment, CommentSortsCache, desc
    from r2.lib.db import tdb_cassandra
    from r2.lib import utils

    # larger has a chance to decrease the number of Cassandra writes,
    # but the probability is low
    chunk_size = 5000

    q = Comment._query(Comment.c._spam == (True, False), Comment.c._deleted == (True, False), sort=desc("_date"))

    if after_id is not None:
        q._after(Comment._byID(after_id))

    q = utils.fetch_things2(q, chunk_size=chunk_size)
    q = utils.progress(q, verbosity=chunk_size, estimate=estimate)

    for chunk in utils.in_chunks(q, chunk_size):
        chunk = filter(lambda x: hasattr(x, "link_id"), chunk)
        update_comment_votes(chunk, write_consistency_level=tdb_cassandra.CL.ONE)
Exemple #23
0
    def perform_actions(self, item, data):
        """Execute all the rule's actions against the item."""
        for key, target in self.targets.iteritems():
            target_item = self.get_target_item(item, data, key)
            target.perform_actions(target_item, data)

        if self.comment:
            comment = self.build_message(self.comment, item, data, disclaimer=True)

            # TODO: shouldn't have to do all this manually
            if isinstance(item, Comment):
                link = data["link"]
                parent_comment = item
            else:
                link = item
                parent_comment = None
            new_comment, inbox_rel = Comment._new(
                ACCOUNT, link, parent_comment, comment, None)
            new_comment.distinguished = "yes"
            new_comment._commit()
            queries.queue_vote(ACCOUNT, new_comment, True, None)
            queries.new_comment(new_comment, inbox_rel)

            g.stats.simple_event("automoderator.comment")

        if self.modmail:
            message = self.build_message(self.modmail, item, data, permalink=True)
            subject = replace_placeholders(
                self.modmail_subject, data, self.matches)
            subject = subject[:100]

            new_message, inbox_rel = Message._new(ACCOUNT, data["subreddit"],
                subject, message, None)
            new_message.distinguished = "yes"
            new_message._commit()
            queries.new_message(new_message, inbox_rel)

            g.stats.simple_event("automoderator.modmail")

        if self.message and not data["author"]._deleted:
            message = self.build_message(self.message, item, data,
                disclaimer=True, permalink=True)
            subject = replace_placeholders(
                self.message_subject, data, self.matches)
            subject = subject[:100]

            new_message, inbox_rel = Message._new(ACCOUNT, data["author"],
                subject, message, None)
            queries.new_message(new_message, inbox_rel)

            g.stats.simple_event("automoderator.message")

        PerformedRulesByThing.mark_performed(item, self)
Exemple #24
0
    def _mock_comment(id=1, author_id=1, link_id=1, sr_id=1, can_comment=True,
                      can_view_promo=True, is_moderator=False, **kwargs):
        kwargs['id'] = id
        kwargs['author_id'] = author_id
        kwargs['link_id'] = link_id

        comment = Comment(**kwargs)
        VByName.run = MagicMock(return_value=comment)

        link = Link(id=link_id)
        Link._byID = MagicMock(return_value=link)

        sr = Subreddit(id=sr_id)
        comment.subreddit_slow = sr
        link.subreddit_slow = sr

        Subreddit.can_comment = MagicMock(return_value=can_comment)
        Link.can_view_promo = MagicMock(return_value=can_view_promo)
        Subreddit.is_moderator = MagicMock(return_value=is_moderator)

        return comment
def _populate(after_id = None, estimate=54301242):
    from r2.models import Comment, CommentSortsCache, desc
    from r2.lib.db import tdb_cassandra
    from r2.lib import utils

    # larger has a chance to decrease the number of Cassandra writes,
    # but the probability is low
    chunk_size = 5000

    q = Comment._query(Comment.c._spam==(True,False),
                       Comment.c._deleted==(True,False),
                       sort=desc('_date'))

    if after_id is not None:
        q._after(Comment._byID(after_id))

    q = utils.fetch_things2(q, chunk_size=chunk_size)
    q = utils.progress(q, verbosity=chunk_size, estimate = estimate)

    for chunk in utils.in_chunks(q, chunk_size):
        chunk = filter(lambda x: hasattr(x, 'link_id'), chunk)
        update_comment_votes(chunk, write_consistency_level = tdb_cassandra.CL.ONE)
Exemple #26
0
def validate_blob(custom):
    """Validate payment_blob and return a dict with everything looked up."""
    ret = {}

    if not custom:
        raise GoldException('no custom')

    payment_blob = g.hardcache.get('payment_blob-%s' % str(custom))
    if not payment_blob:
        raise GoldException('no payment_blob')

    if not ('account_id' in payment_blob and
            'account_name' in payment_blob):
        raise GoldException('no account_id')

    try:
        buyer = Account._byID(payment_blob['account_id'], data=True)
        ret['buyer'] = buyer
    except NotFound:
        raise GoldException('bad account_id')

    if not buyer.name.lower() == payment_blob['account_name'].lower():
        raise GoldException('buyer mismatch')

    goldtype = payment_blob['goldtype']
    ret['goldtype'] = goldtype

    if goldtype == 'gift':
        recipient_name = payment_blob.get('recipient', None)
        if not recipient_name:
            raise GoldException('gift missing recpient')
        try:
            recipient = Account._by_name(recipient_name)
            ret['recipient'] = recipient
        except NotFound:
            raise GoldException('bad recipient')
        comment_fullname = payment_blob.get('comment', None)
        if comment_fullname:
            try:
                ret['comment'] = Comment._by_fullname(comment_fullname)
            except NotFound:
                raise GoldException('bad comment')
        ret['signed'] = payment_blob.get('signed', False)
        giftmessage = payment_blob.get('giftmessage')
        giftmessage = _force_unicode(giftmessage) if giftmessage else None
        ret['giftmessage'] = giftmessage
    elif goldtype not in ('onetime', 'autorenew', 'creddits'):
        raise GoldException('bad goldtype')

    return ret
Exemple #27
0
    def _run_commentstree(msg):
        fname = msg.body
        comment = Comment._by_fullname(fname, data=True)

        link = Link._byID(comment.link_id, data=True)

        try:
            add_comment_tree(comment, link)
        except KeyError:
            # Hackity hack. Try to recover from a corrupted comment
            # tree
            print "Trying to fix broken comments-tree."
            link_comments(link._id, _update=True)
            add_comment_tree(comment, link)
Exemple #28
0
def validate_blob(custom):
    """Validate payment_blob and return a dict with everything looked up."""
    ret = {}

    if not custom:
        raise GoldException('no custom')

    payment_blob = g.hardcache.get('payment_blob-%s' % str(custom))
    if not payment_blob:
        raise GoldException('no payment_blob')

    if not ('account_id' in payment_blob and
            'account_name' in payment_blob):
        raise GoldException('no account_id')

    try:
        buyer = Account._byID(payment_blob['account_id'], data=True)
        ret['buyer'] = buyer
    except NotFound:
        raise GoldException('bad account_id')

    if not buyer.name.lower() == payment_blob['account_name'].lower():
        raise GoldException('buyer mismatch')

    goldtype = payment_blob['goldtype']
    ret['goldtype'] = goldtype

    if goldtype == 'gift':
        recipient_name = payment_blob.get('recipient', None)
        if not recipient_name:
            raise GoldException('gift missing recpient')
        try:
            recipient = Account._by_name(recipient_name)
            ret['recipient'] = recipient
        except NotFound:
            raise GoldException('bad recipient')
        comment_fullname = payment_blob.get('comment', None)
        if comment_fullname:
            try:
                ret['comment'] = Comment._by_fullname(comment_fullname)
            except NotFound:
                raise GoldException('bad comment')
        ret['signed'] = payment_blob.get('signed', False)
        giftmessage = payment_blob.get('giftmessage')
        giftmessage = _force_unicode(giftmessage) if giftmessage else None
        ret['giftmessage'] = giftmessage
    elif goldtype not in ('onetime', 'autorenew', 'creddits'):
        raise GoldException('bad goldtype')

    return ret
Exemple #29
0
def activate_names(link, names):
    for comment, name in names:
        # find a slot to assign a name to. we'll prefer nodes that are
        # currently empty, and failing that find the least-recently-modified
        # node.
        ROOT = "/gold/server-names"
        slot_names = g.zookeeper.get_children(ROOT)
        slots = [(slot_name, g.zookeeper.get(os.path.join(ROOT, slot_name)))
                 for slot_name in slot_names]
        slots.sort(key=lambda (path, (data, stat)): (bool(data), stat.mtime))
        slot_path = os.path.join(ROOT, slots[0][0])

        comment_data = {
            'name': str(name),
            'permalink': comment.make_permalink_slow()
        }
        g.zookeeper.set(slot_path, json.dumps(comment_data))

        lock = g.zookeeper.Lock(slot_path)
        lock_contenders = lock.contenders()
        old_name = lock_contenders[0] if lock_contenders else ""
        old_name = old_name or "one of our servers"

        # reply to the user
        wp = WikiPage.get(SERVERNAME_SR, "templates/success-reply")
        template = random.choice(wp._get("content").split("\r\n---\r\n"))
        comment, inbox_rel = Comment._new(
            author=SYSTEM_ACCOUNT,
            link=link,
            parent=comment,
            body=template % {
                "old-name": old_name,
                "new-name": name,
            },
            ip="127.0.0.1",
        )
        queries.new_comment(comment, inbox_rel)

        # update the link's text
        wp = WikiPage.get(SERVERNAME_SR, "templates/goldisms")
        goldism = random.choice(wp._get("content").split("\r\n---\r\n"))
        wp = WikiPage.get(SERVERNAME_SR, "templates/selftext-success")
        template = wp._get("content")
        link.selftext = template % {
            "old-name": old_name,
            "new-name": name,
            "goldism": goldism,
        }
        link._commit()
Exemple #30
0
def validate_blob(custom):
    """Validate payment_blob and return a dict with everything looked up."""
    ret = {}

    if not custom:
        raise GoldException("no custom")

    payment_blob = g.hardcache.get("payment_blob-%s" % str(custom))
    if not payment_blob:
        raise GoldException("no payment_blob")

    if not ("account_id" in payment_blob and "account_name" in payment_blob):
        raise GoldException("no account_id")

    try:
        buyer = Account._byID(payment_blob["account_id"], data=True)
        ret["buyer"] = buyer
    except NotFound:
        raise GoldException("bad account_id")

    if not buyer.name.lower() == payment_blob["account_name"].lower():
        raise GoldException("buyer mismatch")

    goldtype = payment_blob["goldtype"]
    ret["goldtype"] = goldtype

    if goldtype == "gift":
        recipient_name = payment_blob.get("recipient", None)
        if not recipient_name:
            raise GoldException("gift missing recpient")
        try:
            recipient = Account._by_name(recipient_name)
            ret["recipient"] = recipient
        except NotFound:
            raise GoldException("bad recipient")
        comment_fullname = payment_blob.get("comment", None)
        if comment_fullname:
            try:
                ret["comment"] = Comment._by_fullname(comment_fullname)
            except NotFound:
                raise GoldException("bad comment")
        ret["signed"] = payment_blob.get("signed", False)
        giftmessage = payment_blob.get("giftmessage")
        giftmessage = _force_unicode(giftmessage) if giftmessage else None
        ret["giftmessage"] = giftmessage
    elif goldtype not in ("onetime", "autorenew", "creddits"):
        raise GoldException("bad goldtype")

    return ret
Exemple #31
0
    def _run_commentstree(msg):
        fname = msg.body
        comment = Comment._by_fullname(fname, data=True)

        link = Link._byID(comment.link_id,
                          data=True)

        try:
            add_comment_tree(comment, link)
        except KeyError:
            # Hackity hack. Try to recover from a corrupted comment
            # tree
            print "Trying to fix broken comments-tree."
            link_comments(link._id, _update=True)
            add_comment_tree(comment, link)
def activate_names(link, names):
    for comment, name in names:
        # find a slot to assign a name to. we'll prefer nodes that are
        # currently empty, and failing that find the least-recently-modified
        # node.
        ROOT = "/gold/server-names"
        slot_names = g.zookeeper.get_children(ROOT)
        slots = [(slot_name, g.zookeeper.get(os.path.join(ROOT, slot_name)))
                 for slot_name in slot_names]
        slots.sort(key=lambda (path, (data, stat)): (bool(data), stat.mtime))
        slot_path = os.path.join(ROOT, slots[0][0])

        comment_data = {'name': str(name),
                        'permalink': comment.make_permalink_slow()}
        g.zookeeper.set(slot_path, json.dumps(comment_data))

        lock = g.zookeeper.Lock(slot_path)
        lock_contenders = lock.contenders()
        old_name = lock_contenders[0] if lock_contenders else ""
        old_name = old_name or "one of our servers"

        # reply to the user
        wp = WikiPage.get(SERVERNAME_SR, "templates/success-reply")
        template = random.choice(wp._get("content").split("\r\n---\r\n"))
        comment, inbox_rel = Comment._new(
            author=SYSTEM_ACCOUNT,
            link=link,
            parent=comment,
            body=template % {
                "old-name": old_name,
                "new-name": name,
            },
            ip="127.0.0.1",
        )
        queries.queue_vote(SYSTEM_ACCOUNT, comment, dir=True, ip="127.0.0.1")
        queries.new_comment(comment, inbox_rel)

        # update the link's text
        wp = WikiPage.get(SERVERNAME_SR, "templates/goldisms")
        goldism = random.choice(wp._get("content").split("\r\n---\r\n"))
        wp = WikiPage.get(SERVERNAME_SR, "templates/selftext-success")
        template = wp._get("content")
        link.selftext = template % {
            "old-name": old_name,
            "new-name": name,
            "goldism": goldism,
        }
        link._commit()
def fix_bare_links(apply=False):
    from r2.models import Comment
    from r2.lib.db.thing import NotFound

    fbefore = codecs.open('fix_bare_links_before.txt', 'w', 'utf-8')
    fafter = codecs.open('fix_bare_links_after.txt', 'w', 'utf-8')

    comment_id = 1
    try:
        # The comments are retrieved like this to prevent the API from
        # attempting to load all comments at once and then iterating over them
        while True:
            comment = Comment._byID(comment_id, data=True)

            if (hasattr(comment, 'ob_imported')
                    and comment.ob_imported) and (hasattr(comment, 'is_html')
                                                  and comment.is_html):
                body = comment.body
                if isinstance(body, str):
                    try:
                        body = body.decode('utf-8')
                    except UnicodeDecodeError:
                        print >> sys.stderr, "UnicodeDecodeError, using 'ignore' error mode, comment: %d" % comment._id
                        body = body.decode('utf-8', errors='ignore')
                new_content = rewrite_bare_links(body)

                if new_content != body:
                    print >> fbefore, body
                    print >> fafter, new_content

                    if apply:
                        comment.body = new_content
                        comment._commit()

                    try:
                        print >> sys.stderr, "Rewrote comment %s" % comment.make_permalink_slow(
                        ).encode('utf-8')
                    except UnicodeError:
                        print >> sys.stderr, "Rewrote comment with id: %d" % comment._id

            comment_id += 1
    except NotFound:
        # Assumes that comment ids are sequential and never deleted
        # (which I believe to true) -- wjm
        print >> sys.stderr, "Comment %d not found, exiting" % comment_id

    return
Exemple #34
0
def message_notification_email(data):
    """Queues a system email for a new message notification."""
    from r2.lib.pages import MessageNotificationEmail

    MAX_EMAILS_PER_DAY = 1000
    MESSAGE_THROTTLE_KEY = 'message_notification_emails'

    # If our counter's expired, initialize it again.
    g.cache.add(MESSAGE_THROTTLE_KEY, 0, time=24 * 60 * 60)

    for datum in data.itervalues():
        datum = json.loads(datum)
        user = Account._byID36(datum['to'], data=True)
        comment = Comment._by_fullname(datum['comment'], data=True)

        # In case a user has enabled the preference while it was enabled for
        # them, but we've since turned it off.  We need to explicitly state the
        # user because we're not in the context of an HTTP request from them.
        if not feature.is_enabled('orangereds_as_emails', user=user):
            continue

        if g.cache.get(MESSAGE_THROTTLE_KEY) > MAX_EMAILS_PER_DAY:
            raise Exception(
                'Message notification emails: safety limit exceeded!')

        mac = generate_notification_email_unsubscribe_token(
            datum['to'],
            user_email=user.email,
            user_password_hash=user.password)
        base = g.https_endpoint or g.origin
        unsubscribe_link = base + '/mail/unsubscribe/%s/%s' % (datum['to'],
                                                               mac)

        templateData = {
            'sender_username': datum.get('from', ''),
            'comment': comment,
            'permalink': datum['permalink'],
            'unsubscribe_link': unsubscribe_link,
        }
        _system_email(
            user.email,
            MessageNotificationEmail(**templateData).render(style='email'),
            Email.Kind.MESSAGE_NOTIFICATION,
            from_address=g.notification_email)

        g.stats.simple_event('email.message_notification.queued')
        g.cache.incr(MESSAGE_THROTTLE_KEY)
Exemple #35
0
    def process_comment(self, comment_data, comment, post, comment_dictionary):
        # Prepare data for import
        ip = '127.0.0.1'
        if comment_data:
            naive_date = datetime.datetime.strptime(
                comment_data['dateCreated'], DATE_FORMAT)
            local_date = INPUT_TIMEZONE.localize(
                naive_date, is_dst=False)  # Pick the non daylight savings time
            utc_date = local_date.astimezone(pytz.utc)

            # Determine account to use for this comment
            account = self._get_or_create_account(comment_data['author'],
                                                  comment_data['authorEmail'])

        if comment_data and not comment_data['author'].endswith(
                "| The Effective Altruism Blog"):
            if not comment:
                # Create new comment
                comment, inbox_rel = Comment._new(account,
                                                  post,
                                                  None,
                                                  comment_data['body'],
                                                  ip,
                                                  date=utc_date)
                if str(comment_data['commentParent']) in comment_dictionary:
                    comment.parent_id = comment_dictionary[str(
                        comment_data['commentParent'])]
                comment.is_html = True
                comment.ob_imported = True
                comment._commit()
                comment_dictionary[str(
                    comment_data['commentId'])] = comment._id
            else:
                # Overwrite existing comment
                if str(comment_data['commentParent']) in comment_dictionary:
                    comment.parent_id = comment_dictionary[str(
                        comment_data['commentParent'])]
                comment.author_id = account._id
                comment.body = comment_data['body']
                comment.ip = ip
                comment._date = utc_date
                comment.is_html = True
                comment.ob_imported = True
                comment._commit()
                comment_dictionary[str(
                    comment_data['commentId'])] = comment._id
Exemple #36
0
def comment_exists(post, comment):
    # Check if this comment already exists using brutal compare on content
    # BeautifulSoup is used to parse as HTML in order to remove markup
    content = ''.join(BeautifulSoup(comment['body']).findAll(text=True))
    key = re_non_alphanum.sub('', content)
    existing_comments = Comment._query(Comment.c.link_id == post._id, Comment.c.ob_imported == True, data=True)
    for existing_comment in existing_comments:
        author = Account._byID(existing_comment.author_id, data=True)
        content = ''.join(BeautifulSoup(existing_comment.body).findAll(text=True))
        existing_key = re_non_alphanum.sub('', content)
        if key == existing_key:
            print " Skipping existing %s" % comment_excerpt(comment)
            return True
        # else:
        #     print "%s *|NOT|* %s" % (key, existing_key)

    return False
Exemple #37
0
    def _run_commentstree(msgs, chan):
        fnames = [msg.body for msg in msgs]
        comments = Comment._by_fullname(fnames, data=True, return_dict=False)

        links = Link._byID(set(cm.link_id for cm in comments), data=True, return_dict=True)

        # add the comment to the comments-tree
        for comment in comments:
            l = links[comment.link_id]
            try:
                add_comment_tree(comment, l)
            except KeyError:
                # Hackity hack. Try to recover from a corrupted
                # comment tree
                print "Trying to fix broken comments-tree."
                link_comments(l._id, _update=True)
                add_comment_tree(comment, l)
Exemple #38
0
def _load_link_comments(link_id):
    from r2.models import Comment
    q = Comment._query(Comment.c.link_id == link_id,
                       Comment.c._deleted == (True, False),
                       Comment.c._spam == (True, False),
                       optimize_rules=True,
                       data=True)
    comments = list(q)
    cids = [c._id for c in comments]

    #make a tree
    comment_tree = {}
    for cm in comments:
        p_id = cm.parent_id
        comment_tree.setdefault(p_id, []).append(cm._id)

    #calculate the depths
    depth = {}
    level = 0
    cur_level = comment_tree.get(None, ())
    while cur_level:
        next_level = []
        for cm_id in cur_level:
            depth[cm_id] = level
            next_level.extend(comment_tree.get(cm_id, ()))
        cur_level = next_level
        level += 1

    #calc the number of children
    num_children = {}
    for cm_id in cids:
        num = 0
        todo = [cm_id]
        iteration_count = 0
        while todo:
            if iteration_count > MAX_ITERATIONS:
                raise Exception("bad comment tree for link %s" % link_id)
            more = comment_tree.get(todo.pop(0), ())
            num += len(more)
            todo.extend(more)
            iteration_count += 1
        num_children[cm_id] = num

    num_comments = sum(1 for c in comments if not c._deleted)
    return cids, comment_tree, depth, num_children, num_comments
Exemple #39
0
def _load_link_comments(link_id):
    from r2.models import Comment
    q = Comment._query(Comment.c.link_id == link_id,
                       Comment.c._deleted == (True, False),
                       Comment.c._spam == (True, False),
                       optimize_rules=True,
                       data = True)
    comments = list(q)
    cids = [c._id for c in comments]

    #make a tree
    comment_tree = {}
    for cm in comments:
        p_id = cm.parent_id
        comment_tree.setdefault(p_id, []).append(cm._id)

    #calculate the depths
    depth = {}
    level = 0
    cur_level = comment_tree.get(None, ())
    while cur_level:
        next_level = []
        for cm_id in cur_level:
            depth[cm_id] = level
            next_level.extend(comment_tree.get(cm_id, ()))
        cur_level = next_level
        level += 1

    #calc the number of children
    num_children = {}
    for cm_id in cids:
        num = 0
        todo = [cm_id]
        iteration_count = 0
        while todo:
            if iteration_count > MAX_ITERATIONS:
                raise Exception("bad comment tree for link %s" % link_id)
            more = comment_tree.get(todo.pop(0), ())
            num += len(more)
            todo.extend(more)
            iteration_count += 1
        num_children[cm_id] = num

    num_comments = sum(1 for c in comments if not c._deleted)
    return cids, comment_tree, depth, num_children, num_comments
Exemple #40
0
def fix_bare_links(apply=False):
    from r2.models import Comment
    from r2.lib.db.thing import NotFound
    
    fbefore = codecs.open('fix_bare_links_before.txt', 'w', 'utf-8')
    fafter  = codecs.open('fix_bare_links_after.txt', 'w', 'utf-8')
    
    comment_id = 1
    try:
        # The comments are retrieved like this to prevent the API from 
        # attempting to load all comments at once and then iterating over them
        while True:
            comment = Comment._byID(comment_id, data=True)
        
            if (hasattr(comment, 'ob_imported') and comment.ob_imported) and (hasattr(comment, 'is_html') and comment.is_html):
                body = comment.body
                if isinstance(body, str):
                    try:
                        body = body.decode('utf-8')
                    except UnicodeDecodeError:
                        print >>sys.stderr, "UnicodeDecodeError, using 'ignore' error mode, comment: %d" % comment._id
                        body = body.decode('utf-8', errors='ignore')
                new_content = rewrite_bare_links(body)
                
                if new_content != body:
                    print >>fbefore, body
                    print >>fafter, new_content
                    
                    if apply:
                        comment.body = new_content
                        comment._commit()
                    
                    try:
                        print >>sys.stderr, "Rewrote comment %s" % comment.make_permalink_slow().encode('utf-8')
                    except UnicodeError:
                        print >>sys.stderr, "Rewrote comment with id: %d" % comment._id
                    
            
            comment_id += 1
    except NotFound:
        # Assumes that comment ids are sequential and never deleted
        # (which I believe to true) -- wjm
        print >>sys.stderr, "Comment %d not found, exiting" % comment_id

    return
Exemple #41
0
def message_notification_email(data):
    """Queues a system email for a new message notification."""
    from r2.lib.pages import MessageNotificationEmail

    MAX_EMAILS_PER_DAY = 1000
    MESSAGE_THROTTLE_KEY = 'message_notification_emails'

    # If our counter's expired, initialize it again.
    g.cache.add(MESSAGE_THROTTLE_KEY, 0, time=24*60*60)

    for datum in data.itervalues():
        datum = json.loads(datum)
        user = Account._byID36(datum['to'], data=True)
        comment = Comment._by_fullname(datum['comment'], data=True)

        # In case a user has enabled the preference while it was enabled for
        # them, but we've since turned it off.  We need to explicitly state the
        # user because we're not in the context of an HTTP request from them.
        if not feature.is_enabled('orangereds_as_emails', user=user):
            continue

        if g.cache.get(MESSAGE_THROTTLE_KEY) > MAX_EMAILS_PER_DAY:
            raise Exception(
                    'Message notification emails: safety limit exceeded!')

        mac = generate_notification_email_unsubscribe_token(
                datum['to'], user_email=user.email,
                user_password_hash=user.password)
        base = g.https_endpoint or g.origin
        unsubscribe_link = base + '/mail/unsubscribe/%s/%s' % (datum['to'], mac)

        templateData = {
            'sender_username': datum.get('from', ''),
            'comment': comment,
            'permalink': datum['permalink'],
            'unsubscribe_link': unsubscribe_link,
        }
        _system_email(user.email,
                      MessageNotificationEmail(**templateData).render(style='email'),
                      Email.Kind.MESSAGE_NOTIFICATION,
                      from_address=g.notification_email)

        g.stats.simple_event('email.message_notification.queued')
        g.cache.incr(MESSAGE_THROTTLE_KEY)
Exemple #42
0
    def _run_commentstree(msgs, chan):
        fnames = [msg.body for msg in msgs]
        comments = Comment._by_fullname(fnames, data=True, return_dict=False)

        links = Link._byID(set(cm.link_id for cm in comments),
                           data=True,
                           return_dict=True)

        # add the comment to the comments-tree
        for comment in comments:
            l = links[comment.link_id]
            try:
                add_comment_tree(comment, l)
            except KeyError:
                # Hackity hack. Try to recover from a corrupted
                # comment tree
                print "Trying to fix broken comments-tree."
                link_comments(l._id, _update=True)
                add_comment_tree(comment, l)
Exemple #43
0
    def comment_event(self, new_comment, request=None, context=None):
        """Create a 'comment' event for event-collector.

        new_comment: An r2.models.Comment object
        request, context: Should be pylons.request & pylons.c respectively
        """
        from r2.models import Comment, Link

        event = Event(
            topic="comment_events",
            event_type="ss.comment",
            time=new_comment._date,
            request=request,
            context=context,
            truncatable_field="comment_body",
        )

        event.add("comment_id", new_comment._id)
        event.add("comment_fullname", new_comment._fullname)

        event.add_text("comment_body", new_comment.body)

        post = Link._byID(new_comment.link_id)
        event.add("post_id", post._id)
        event.add("post_fullname", post._fullname)
        event.add("post_created_ts", to_epoch_milliseconds(post._date))
        if post.promoted:
            event.add("post_is_promoted", bool(post.promoted))

        if new_comment.parent_id:
            parent = Comment._byID(new_comment.parent_id)
        else:
            # If this is a top-level comment, parent is the same as the post
            parent = post
        event.add("parent_id", parent._id)
        event.add("parent_fullname", parent._fullname)
        event.add("parent_created_ts", to_epoch_milliseconds(parent._date))

        event.add("user_neutered", new_comment.author_slow._spam)

        event.add_subreddit_fields(new_comment.subreddit_slow)

        self.save_event(event)
def fix_images(dryrun=True):
    from r2.models import Link, Comment

    links = Link._query(Link.c.ob_permalink != None, data = True)
    for link in links:
        ob_url = link.ob_permalink.strip()
        print "Processing %s" % ob_url

        new_content = process_content(link.article)
        if not dryrun:
            link.article = new_content
            link._commit()

        comments = Comment._query(Comment.c.link_id == link._id, data = True)
        for comment in comments:
            new_content = process_content(comment.body)
            if not dryrun:
                comment.body = new_content
                comment._commit()
    def comment_event(self, new_comment, request=None, context=None):
        """Create a 'comment' event for event-collector.

        new_comment: An r2.models.Comment object
        request, context: Should be pylons.request & pylons.c respectively
        """
        from r2.models import Comment, Link

        event = Event(
            topic="comment_events",
            event_type="ss.comment",
            time=new_comment._date,
            request=request,
            context=context,
            truncatable_field="comment_body",
        )

        event.add("comment_id", new_comment._id)
        event.add("comment_fullname", new_comment._fullname)

        event.add_text("comment_body", new_comment.body)

        post = Link._byID(new_comment.link_id)
        event.add("post_id", post._id)
        event.add("post_fullname", post._fullname)
        event.add("post_created_ts", to_epoch_milliseconds(post._date))
        if post.promoted:
            event.add("post_is_promoted", bool(post.promoted))

        if new_comment.parent_id:
            parent = Comment._byID(new_comment.parent_id)
        else:
            # If this is a top-level comment, parent is the same as the post
            parent = post
        event.add("parent_id", parent._id)
        event.add("parent_fullname", parent._fullname)
        event.add("parent_created_ts", to_epoch_milliseconds(parent._date))

        event.add("user_neutered", new_comment.author_slow._spam)

        event.add_subreddit_fields(new_comment.subreddit_slow)

        self.save_event(event)
def fix_images(dryrun=True):
    from r2.models import Link, Comment

    links = Link._query(Link.c.ob_permalink != None, data=True)
    for link in links:
        ob_url = link.ob_permalink.strip()
        print "Processing %s" % ob_url

        new_content = process_content(link.article)
        if not dryrun:
            link.article = new_content
            link._commit()

        comments = Comment._query(Comment.c.link_id == link._id, data=True)
        for comment in comments:
            new_content = process_content(comment.body)
            if not dryrun:
                comment.body = new_content
                comment._commit()
Exemple #47
0
    def on_use(self, user, target):
        link = Link._byID(target.link_id)
        comment_tree = get_comment_tree(link)
        child_ids = comment_tree.tree[target._id]
        grandchild_ids = []
        for child_id in child_ids:
            grandchild_ids.extend(comment_tree.tree[child_id])

        comments = Comment._byID(child_ids + grandchild_ids, data=True,
                                 return_dict=True)
        children = [comments[cid] for cid in child_ids]
        grandchildren = [comments[cid] for cid in grandchild_ids]

        for comment in itertools.chain([target], children, grandchildren):
            effects.add_effect(user, comment, self.item_name)

        self.apply_damage_and_log(user, [target], self.direct_damage)
        self.apply_damage_and_log(user, children, self.child_damage)
        self.apply_damage_and_log(user, grandchildren, self.grandchild_damage)
def _load_br_criticisms(link_id):
    from r2.models import Comment
    q = Comment._query(Comment.c.link_id == link_id,
                       Comment.c._deleted == (True, False),
                       Comment.c._spam == (True, False),
                       optimize_rules=True,
                       data = True)
    comments = list(q)
    brs = [c for c in comments if c.bestresponse==True]
    #print "SQUIRRREL!"
    #print comments
    #print link_id
    outs = [c._id for c in brs]
    ups = [c.parent_id for c in brs if c.parent_id]
    while ups:
    	brs = [c for c in comments if c._id in ups]
        outs.extend([c._id for c in brs])
        ups = [c.parent_id for c in brs if c.parent_id]
    
    return outs
Exemple #49
0
    def on_use(self, user, target):
        link = Link._byID(target.link_id)
        comment_tree = get_comment_tree(link)
        child_ids = comment_tree.tree[target._id]
        grandchild_ids = []
        for child_id in child_ids:
            grandchild_ids.extend(comment_tree.tree[child_id])

        comments = Comment._byID(child_ids + grandchild_ids,
                                 data=True,
                                 return_dict=True)
        children = [comments[cid] for cid in child_ids]
        grandchildren = [comments[cid] for cid in grandchild_ids]

        for comment in itertools.chain([target], children, grandchildren):
            effects.add_effect(user, comment, self.item_name)

        self.apply_damage_and_log(user, [target], self.direct_damage)
        self.apply_damage_and_log(user, children, self.child_damage)
        self.apply_damage_and_log(user, grandchildren, self.grandchild_damage)
Exemple #50
0
    def spam_account_comments(self, account, query_limit=10000, spam_limit=500):
        from r2.lib.db.operators import asc, desc, timeago

        q = Comment._query(Comment.c.author_id == account._id,
            Link.c._spam == False,
            sort=desc('_date'),
            data=False)
        q._limit = query_limit
        things = list(q)

        processed = 0
        for item in things:
            if processed < spam_limit:
                verdict = getattr(item, "verdict", None)
                if not verdict or not verdict.endswith("-approved"):
                    processed += 1
                    admintools.spam(item,
                        auto=False,
                        moderator_banned=False,
                        banner=None,
                        train_spam=True)
Exemple #51
0
def _load_link_comments(link_id):
    from r2.models import Comment
    q = Comment._query(
        Comment.c.link_id == link_id,
        Comment.c._deleted == (True, False),
        Comment.c._spam == (True, False),
        data=True)
    comments = list(q)
    cids = [c._id for c in comments]

    #make a tree
    comment_tree = {}
    for cm in comments:
        p_id = cm.parent_id
        comment_tree.setdefault(p_id, []).append(cm._id)

    #calculate the depths
    depth = {}
    level = 0
    cur_level = comment_tree.get(None, ())
    while cur_level:
        next_level = []
        for cm_id in cur_level:
            depth[cm_id] = level
            next_level.extend(comment_tree.get(cm_id, ()))
        cur_level = next_level
        level += 1

    #calc the number of children
    num_children = {}
    for cm_id in cids:
        num = 0
        todo = [cm_id]
        while todo:
            more = comment_tree.get(todo.pop(0), ())
            num += len(more)
            todo.extend(more)
        num_children[cm_id] = num

    return cids, comment_tree, depth, num_children
Exemple #52
0
def _load_link_comments(link_id):
    from r2.models import Comment
    q = Comment._query(Comment.c.link_id == link_id,
                       Comment.c._deleted == (True, False),
                       Comment.c._spam == (True, False),
                       data=True)
    comments = list(q)
    cids = [c._id for c in comments]

    #make a tree
    comment_tree = {}
    for cm in comments:
        p_id = cm.parent_id
        comment_tree.setdefault(p_id, []).append(cm._id)

    #calculate the depths
    depth = {}
    level = 0
    cur_level = comment_tree.get(None, ())
    while cur_level:
        next_level = []
        for cm_id in cur_level:
            depth[cm_id] = level
            next_level.extend(comment_tree.get(cm_id, ()))
        cur_level = next_level
        level += 1

    #calc the number of children
    num_children = {}
    for cm_id in cids:
        num = 0
        todo = [cm_id]
        while todo:
            more = comment_tree.get(todo.pop(0), ())
            num += len(more)
            todo.extend(more)
        num_children[cm_id] = num

    return cids, comment_tree, depth, num_children
Exemple #53
0
def _get_sr_comments(sr_id):
    """the subreddit /r/foo/comments page"""
    q = Comment._query(Comment.c.sr_id == sr_id,
                       sort = desc('_date'))
    return make_results(q)
Exemple #54
0
def get_all_comments():
    """the master /comments page"""
    q = Comment._query(sort = desc('_date'))
    return make_results(q)
Exemple #55
0
def link_comments_and_sort(link_id, sort):
    from r2.models import Comment, CommentSortsCache

    # This has grown sort of organically over time. Right now the
    # cache of the comments tree consists in three keys:
    # 1. The comments_key: A tuple of
    #      (cids, comment_tree, depth, num_children)
    #    given:
    #      cids         =:= [comment_id]
    #      comment_tree =:= dict(comment_id -> [comment_id])
    #      depth        =:= dict(comment_id -> int depth)
    #      num_children =:= dict(comment_id -> int num_children)
    # 2. The parent_comments_key =:= dict(comment_id -> parent_id)
    # 3. The comments_sorts keys =:= dict(comment_id36 -> float).
    #    These are represented by a Cassandra model
    #    (CommentSortsCache) rather than a permacache key. One of
    #    these exists for each sort (hot, new, etc)

    # performance hack: preload these into the LocalCache at the same
    # time
    g.permacache.get_multi(
        [comments_key(link_id),
         parent_comments_key(link_id)])

    cids, cid_tree, depth, num_children = link_comments(link_id)

    # load the sorter
    sorter = _get_comment_sorter(link_id, sort)

    sorter_needed = []
    if cids and not sorter:
        sorter_needed = cids
        g.log.debug("comment_tree.py: sorter (%s) cache miss for Link %s" %
                    (sort, link_id))
        sorter = {}

    sorter_needed = [x for x in cids if x not in sorter]
    if cids and sorter_needed:
        g.log.debug(
            "Error in comment_tree: sorter %r inconsistent (missing %d e.g. %r)"
            % (sort_comments_key(
                link_id, sort), len(sorter_needed), sorter_needed[:10]))
        if not g.disallow_db_writes:
            update_comment_votes(
                Comment._byID(sorter_needed, data=True, return_dict=False))

        sorter.update(_comment_sorter_from_cids(sorter_needed, sort))

    # load the parents
    key = parent_comments_key(link_id)
    parents = g.permacache.get(key)
    if parents is None:
        g.log.debug("comment_tree.py: parents cache miss for Link %s" %
                    link_id)
        parents = {}
    elif cids and not all(x in parents for x in cids):
        g.log.debug("Error in comment_tree: parents inconsistent for Link %s" %
                    link_id)
        parents = {}

    if not parents and len(cids) > 0:
        with g.make_lock(lock_key(link_id)):
            # reload from the cache so the sorter and parents are
            # maximally consistent
            r = g.permacache.get(comments_key(link_id))
            cids, cid_tree, depth, num_children = r

            key = parent_comments_key(link_id)
            if not parents:
                parents = _parent_dict_from_tree(cid_tree)
                g.permacache.set(key, parents)

    return cids, cid_tree, depth, num_children, parents, sorter
Exemple #56
0
    Subreddit,
)

LINK_GILDING_START = datetime(2014, 2, 1, 0, 0, tzinfo=g.tz)
COMMENT_GILDING_START = datetime(2012, 10, 1, 0, 0, tzinfo=g.tz)

queries = [
    Link._query(
        Link.c.gildings != 0,
        Link.c._date > LINK_GILDING_START,
        data=True,
        sort=desc('_date'),
    ),
    Comment._query(
        Comment.c.gildings != 0,
        Comment.c._date > COMMENT_GILDING_START,
        data=True,
        sort=desc('_date'),
    ),
]

seconds_by_srid = defaultdict(int)
gilding_price = g.gold_month_price.pennies

for q in queries:
    for things in fetch_things2(q, chunks=True, chunk_size=100):
        print things[0]._fullname

        for thing in things:
            seconds_per_gilding = calculate_server_seconds(
                gilding_price, thing._date)
            seconds_by_srid[thing.sr_id] += int(thing.gildings *