示例#1
0
    def on_status(self, status):
        if (
            db.session.query(Tweet).filter_by(tweet_id=status.id).count() == 0
            and
            # TODO(Need to encode this into unicode.)
            len(str(status.text.encode("unicode_escape"))) < 1000
        ):
            LOGGING.push("*" + status.user.name + "*: " + LOGGING.clean(status.text))

            Tweet.store_tweet(status)

            self.num_tweets += 1

            if self.num_tweets % 100 == 0:
                LOGGING.push("*" + str(self.num_tweets) + "* tweets have been collected.")
示例#2
0
def crawl_category(category):
    """Crawls a specific given category.

    Args:
        category: Category to search for.
    """

    cursor = limit_handled(
        tweepy.Cursor(API.search, q=category, count=100).items(1000)
    )

    for status in cursor:
        if (
            db.session.query(Tweet).filter_by(
                tweet_id=status.id
            ).count() == 0 and
            # TODO(Need to encode this into unicode.)
            len(str(status.text.encode('unicode_escape'))) < 1000
        ):
            LOGGING.push(
                "*" + status.user.name + "*: " + LOGGING.clean(status.text)
            )

            Tweet.store_tweet(status)