Beispiel #1
0
    def test_send_to_database(self):
        type = Type.TEST
        add_data(session=get_engine_session(),
                 entry_type=type.value,
                 handled_utc=(datetime.now().strftime('%Y-%m-%d %H:%M:%S')),
                 original_url="https://google.com/thisisatest",
                 canonical_url=None,
                 note="Test")

        self.assertTrue(True)
def run_bot(type=Type.MENTION,
            guess_and_check=True,
            reply_to_item=True,
            write_to_database=True):
    # Get the stream instance (contains session, type and data)
    s = stream.get_stream(type)
    log.info("Set up new stream")

    # Start the stream
    for message in s.praw_session.inbox.stream():
        # Mark the item as read
        message.mark_read()

        # Log the message type and id
        log.info(f"New message: {message.type}: {message.fullname}")

        # If the message is a comment_reply, ignore it
        if message.type == "comment_reply":
            continue
        # If the message is an username_mention, start summon process
        if message.type == "username_mention":
            parent = message.parent()
            i = Item(type=Type.MENTION,
                     id=parent.name,
                     subreddit=parent.subreddit,
                     author=parent.author,
                     context=message.context,
                     summoner=message.author,
                     parent_link=parent.permalink)

            # Check if the parent is a comment or submission
            if isinstance(parent, praw.models.Comment):
                i.body = parent.body
                i.parent_type = Type.COMMENT
            elif isinstance(parent, praw.models.Submission):
                i.body = get_submission_body(parent)
                i.parent_type = Type.SUBMISSION
            else:
                log.warning("Unknown parent instance")

            # Check if the item meets the criteria
            meets_criteria, result_code = check_criteria(
                item=i,
                data=s,
                history_failed=s.mentions_failed,
                history_success=s.mentions_success,
                mustBeAMP=True,
                mustBeNew=True,
                mustNotBeDisallowedSubreddit=True,
                mustNotHaveFailed=True,
                mustNotBeMine=True,
                mustNotBeOptedOut=True,
                mustNotHaveDisallowedMods=True)

            # If it meets the criteria, try to find the canonicals and make a reply
            if result_code != ResultCode.ERROR_NO_AMP:
                log.info(
                    f"{i.id} in r/{i.subreddit} is AMP, result_code={result_code.value}"
                )
                # Get the urls from the body and try to find the canonicals
                urls = get_urls(i.body)
                i.links = get_urls_info(urls, guess_and_check)

                # If a canonical was found, generate a reply, otherwise log a warning
                if any(link.canonical
                       for link in i.links) or any(link.amp_canonical
                                                   for link in i.links):
                    # Generate a reply
                    reply_text, reply_canonical_text = generate_reply(
                        stream_type=s.type,
                        np_subreddits=s.np_subreddits,
                        item_type=i.parent_type,
                        subreddit=i.subreddit,
                        links=i.links,
                        summoned_link=i.context)

                    # Send a DM if AmputatorBot can't reply because it's disallowed by a subreddit, mod or user
                    if result_code == ResultCode.ERROR_DISALLOWED_SUBREDDIT \
                            or result_code == ResultCode.ERROR_DISALLOWED_MOD \
                            or result_code == ResultCode.ERROR_USER_OPTED_OUT:

                        # Generate and send an error DM dynamically based on the error
                        subject, message = dm_generator(
                            result_code=result_code,
                            parent_link=i.parent_link,
                            parent_subreddit=i.subreddit,
                            parent_type=i.parent_type.value,
                            first_amp_url=i.links[0].url_clean,
                            canonical_text=reply_canonical_text)
                        s.praw_session.redditor(str(i.summoner)).message(
                            subject, message)
                        log.info(f"Send summoner DM of type {result_code}")

                    # Try to post the reply, send a DM to the summoner
                    elif reply_to_item:
                        try:
                            reply = parent.reply(reply_text)
                            log.info(f"Replied to {i.id} with {reply.name}")
                            update_local_data("mentions_success", i.id)
                            s.mentions_success.append(i.id)

                            # Generate and send a SUCCESS DM to the summoner
                            result_code = ResultCode.SUCCESS
                            subject, message = dm_generator(
                                result_code=result_code,
                                reply_link=reply.permalink,
                                parent_subreddit=i.subreddit,
                                parent_type=i.parent_type.value,
                                parent_link=i.parent_link,
                                first_amp_url=i.links[0].url_clean,
                                canonical_text=reply_canonical_text)
                            s.praw_session.redditor(str(i.summoner)).message(
                                subject, message)
                            log.info(f"Send summoner DM of type {result_code}")

                        except (Forbidden, Exception):
                            log.warning("Couldn't post reply!")
                            log.error(traceback.format_exc())
                            update_local_data("mentions_failed", i.id)
                            s.mentions_failed.append(i.id)

                            # Check if AmputatorBot is banned in the subreddit
                            is_banned = check_if_banned(i.subreddit)
                            if is_banned:
                                update_local_data("disallowed_subreddits",
                                                  i.subreddit)
                                s.disallowed_subreddits.append(i.subreddit)

                            # Generate and send an ERROR_REPLY_FAILED DM to the summoner
                            result_code = ResultCode.ERROR_REPLY_FAILED
                            subject, message = dm_generator(
                                result_code=result_code,
                                parent_type=i.parent_type.value,
                                parent_link=i.parent_link,
                                first_amp_url=i.links[0].url_clean,
                                canonical_text=reply_canonical_text)
                            s.praw_session.redditor(str(i.summoner)).message(
                                subject, message)
                            log.info(f"Send summoner DM of type {result_code}")

                # If no canonicals were found, log the failed attempt
                else:
                    log.warning("No canonicals found")
                    update_local_data("mentions_failed", i.id)
                    s.mentions_failed.append(i.id)

                    # Check if the domain is problematic (meaning it's raising frequent errors)
                    if any(link.domain in s.problematic_domains
                           for link in i.links):
                        result_code = ResultCode.ERROR_PROBLEMATIC_DOMAIN
                    else:
                        result_code = ResultCode.ERROR_NO_CANONICALS

                    # Generate and send an
                    subject, message = dm_generator(
                        result_code=result_code,
                        parent_type=i.parent_type.value,
                        parent_link=i.parent_link,
                        first_amp_url=i.links[0].url_clean)

                    s.praw_session.redditor(str(i.summoner)).message(
                        subject, message)

                # If write_to_database is enabled, make a new entry for every URL
                if write_to_database:
                    for link in i.links:
                        if link.is_amp:
                            add_data(session=get_engine_session(),
                                     entry_type=type.value,
                                     handled_utc=datetime.now().strftime(
                                         '%Y-%m-%d %H:%M:%S'),
                                     original_url=link.url_clean,
                                     canonical_url=link.canonical)

        # If the message is a DM / message, check for opt-out and opt-back-in requests
        elif message.type == "unknown":
            subject = message.subject.lower()
            if subject == "opt me out of amputatorbot":
                try:
                    author = message.author.name
                    log.info(f"New opt-out request by {author}")

                    # If the user is already opted out, notify the user
                    if author.casefold() in list(
                            user.casefold() for user in s.disallowed_users):
                        log.warning("User has already opted out!")
                        s.praw_session.redditor(author).message(
                            subject=
                            "You have already opted out of AmputatorBot",
                            message=
                            "You have already opted out, so AmputatorBot won't reply to your comments "
                            "and submissions anymore. You will still be able to see AmputatorBot's replies to "
                            "other people's content. Block u/AmputatorBot if you don't want that either. "
                            "Cheers!")

                    # If the user hasn't been opted out yet, add user to the list and notify the user
                    else:
                        log.info("User has not opted out yet")
                        update_local_data("disallowed_users", author)
                        s.disallowed_users.append(author)
                        s.praw_session.redditor(author).message(
                            subject=
                            "You have successfully opted out of AmputatorBot",
                            message=
                            "You have successfully opted out of AmputatorBot. AmputatorBot won't reply to your "
                            "comments and submissions anymore (although it can take up to 24 hours to fully "
                            "process your opt-out request). You will still be able to see AmputatorBot's "
                            "replies to other people's content. Block u/AmputatorBot if you don't want that "
                            "either. Cheers!")

                except (RedditAPIException, Forbidden, Exception):
                    log.error(traceback.format_exc())
                    log.warning(
                        f"Something went wrong while processing opt-out request {message.fullname}"
                    )

            elif subject == "opt me back in again of amputatorbot":
                try:
                    author = message.author.name
                    log.info(f"New opt-back-in request by {author}")

                    # If the user is not opted out, notify the user
                    if author.casefold() not in list(
                            user.casefold() for user in s.disallowed_users):
                        log.warning("User is not opted out!")
                        s.praw_session.redditor(author).message(
                            subject="You don't have to opt in of AmputatorBot",
                            message=
                            "This opt-back-in feature is meant only for users who choose to opt-out earlier "
                            "but now regret it. At no point did you opt out of AmputatorBot so there's no "
                            "need to opt back in. Cheers!")

                    # If the user has opted out, remove user from the list and notify the user
                    else:
                        log.info("User is currently opted out")
                        remove_local_data("disallowed_users", author)
                        s.disallowed_users.remove(author)
                        s.praw_session.redditor(author).message(
                            subject=
                            "You have successfully opted back in of AmputatorBot",
                            message=
                            "You have successfully opted back in of AmputatorBot, meaning AmputatorBot can "
                            "reply to your comments and submissions again (although it can take up to 24 hours "
                            "to fully process your opt-back-in request). Thank you! Cheers!"
                        )

                except (RedditAPIException, Forbidden, Exception):
                    log.error(traceback.format_exc())
                    log.warning(
                        f"Something went wrong while processing opt-back-in request {message.fullname}"
                    )

            elif "you've been permanently banned from participating in" in subject:
                subreddit = message.subreddit
                if subreddit:
                    log.info(f"New ban issued by r/{subreddit}")
                    is_banned = check_if_banned(subreddit)
                    if is_banned:
                        update_local_data("disallowed_subreddits", subreddit)
                        s.disallowed_subreddits.append(subreddit)
                        log.info(f"Added {subreddit} to disallowed_subreddits")
                else:
                    log.warning(
                        f"Message wasn't send by a subreddit, but by {message.author.name}"
                    )

        else:
            log.warning(f"Unknown message type: {message.type}")
            continue
        log.info("\n")
Beispiel #3
0
def run_bot(type=Type.SUBMISSION,
            guess_and_check=False,
            reply_to_post=True,
            write_to_database=True):
    # Get the stream instance (contains session, type and data)
    s = stream.get_stream(type)
    log.info("Set up new stream")

    # Start the stream
    for submission in s.praw_session.subreddit("+".join(
            s.allowed_subreddits)).stream.submissions():
        # Generate an item with all the relevant data
        i = Item(type=type,
                 id=submission.name,
                 subreddit=submission.subreddit,
                 author=submission.author,
                 body=get_submission_body(submission))

        # Check if the item meets the criteria
        meets_criteria, result_code = check_criteria(
            item=i,
            data=s,
            history_failed=s.submissions_failed,
            history_success=s.submissions_success,
            mustBeAMP=True,
            mustBeNew=True,
            mustNotBeDisallowedSubreddit=False,
            mustNotHaveFailed=True,
            mustNotBeMine=True,
            mustNotBeOptedOut=True,
            mustNotHaveDisallowedMods=False)

        # If it meets the criteria, try to find the canonicals and make a reply
        if meets_criteria:
            log.info(f"{i.id} in r/{i.subreddit} meets criteria")
            # Get the urls from the body and try to find the canonicals
            urls = get_urls(i.body)
            i.links = get_urls_info(urls, guess_and_check)

            # If a canonical was found, generate a reply, otherwise log a warning
            if any(link.canonical
                   for link in i.links) or any(link.amp_canonical
                                               for link in i.links):
                # Generate a reply
                reply_text, reply_canonical_text = generate_reply(
                    stream_type=s.type,
                    np_subreddits=s.np_subreddits,
                    item_type=i.type,
                    links=i.links,
                    subreddit=i.subreddit)

                # Try to post the reply
                if reply_to_post:
                    try:
                        reply = submission.reply(reply_text)
                        log.info(f"Replied to {i.id} with {reply.name}")
                        update_local_data("submissions_success", i.id)
                        s.submissions_success.append(i.id)

                    except (Forbidden, Exception):
                        log.warning("Couldn't post reply!")
                        log.error(traceback.format_exc())
                        update_local_data("submissions_failed", i.id)
                        s.submissions_failed.append(i.id)

                        # Check if AmputatorBot is banned in the subreddit
                        is_banned = check_if_banned(i.subreddit)
                        if is_banned:
                            update_local_data("disallowed_subreddits",
                                              i.subreddit)
                            s.disallowed_subreddits.append(i.subreddit)

            # If no canonicals were found, log the failed attempt
            else:
                log.warning("No canonicals found")
                update_local_data("submissions_failed", i.id)
                s.submissions_failed.append(i.id)

            # If write_to_database is enabled, make a new entry for every URL
            if write_to_database:
                for link in i.links:
                    if link.is_amp:
                        add_data(session=get_engine_session(),
                                 entry_type=type.value,
                                 handled_utc=datetime.now().strftime(
                                     '%Y-%m-%d %H:%M:%S'),
                                 original_url=link.url_clean,
                                 canonical_url=link.canonical)
    def on_status(self, data):
        try:
            i = Item(type=Type.TWEET,
                     id=data.id,
                     body=data.text.encode(encoding='utf-8',
                                           errors='ignore').decode(),
                     author=get_twitterer_name(data))

            cached_urls = get_cached_tweet_urls(data)

            if len(cached_urls) >= 1:
                # Check if the item meets the criteria
                meets_criteria, result_code = check_tweet_criteria(
                    item=i,
                    cached_urls=cached_urls,
                    tweet=data,
                    data=self.s,
                    history_failed=self.s.tweets_failed,
                    history_success=self.s.tweets_success,
                    mustBeAMP=True,
                    mustNotBeRetweet=True,
                    mustBeCached=True,
                    mustBeNew=True,
                    mustNotHaveFailed=True,
                    mustNotBeMine=True,
                    mustNotBeOptedOut=True)

                # If it meets the criteria, try to find the canonicals and make a reply
                if meets_criteria:
                    log.info(f"{i.id} meets criteria")
                    # Get the urls from the body and try to find the canonicals
                    i.links = get_urls_info(cached_urls, self.guess_and_check)

                    # If a canonical was found, generate a reply, otherwise log a warning
                    if any(link.canonical for link in i.links):
                        # Generate a reply
                        generated_tweet = generate_tweet(i.links)

                        # Try to post the reply
                        if self.reply_to_item:
                            try:
                                reply = self.api.update_status(
                                    status=generated_tweet,
                                    in_reply_to_status_id=i.id,
                                    auto_populate_reply_metadata=True)
                                log.info(f"Replied to {i.id} with {reply.id}")
                                update_local_data("tweets_success", i.id)
                                self.s.tweets_success.append(i.id)

                            except (TweepError, Exception):
                                log.warning("Couldn't post reply!")
                                log.error(traceback.format_exc())
                                update_local_data("tweets_failed", i.id)
                                self.s.tweets_failed.append(i.id)

                    # If no canonicals were found, log the failed attempt
                    else:
                        log.warning("No canonicals found")
                        update_local_data("tweets_failed", i.id)
                        self.s.tweets_failed.append(i.id)

                    # If write_to_database is enabled, make a new entry for every URL
                    if self.write_to_database:
                        for link in i.links:
                            add_data(session=get_engine_session(),
                                     entry_type=i.type.value,
                                     handled_utc=datetime.now().strftime(
                                         '%Y-%m-%d %H:%M:%S'),
                                     original_url=link.url_clean,
                                     canonical_url=link.canonical)

        except (TweepError, Exception):
            log.error(traceback.format_exc())
            log.warning("\nSomething went wrong while handling a tweet")
            sleep(120)
        return True