def on_status(self, status):
if (
db.session.query(Tweet).filter_by(tweet_id=status.id).count() == 0
and
# TODO(Need to encode this into unicode.)
len(str(status.text.encode("unicode_escape"))) < 1000
):
LOGGING.push("*" + status.user.name + "*: " + LOGGING.clean(status.text))
Tweet.store_tweet(status)
self.num_tweets += 1
if self.num_tweets % 100 == 0:
LOGGING.push("*" + str(self.num_tweets) + "* tweets have been collected.")
def crawl_category(category):
"""Crawls a specific given category.
Args:
category: Category to search for.
"""
cursor = limit_handled(
tweepy.Cursor(API.search, q=category, count=100).items(1000)
)
for status in cursor:
if (
db.session.query(Tweet).filter_by(
tweet_id=status.id
).count() == 0 and
# TODO(Need to encode this into unicode.)
len(str(status.text.encode('unicode_escape'))) < 1000
):
LOGGING.push(
"*" + status.user.name + "*: " + LOGGING.clean(status.text)
)
Tweet.store_tweet(status)