def load_tweets(profile, api):
tweets = []
for status in tweepy.Cursor(api.user_timeline).items():
print str(status.id)
tweet = Tweet(temp=str(status.id)[-9:])
tweet.text = unicode(status.text)
tweet.created = timezone.make_aware(status.created_at, timezone.get_current_timezone())
tweet.is_retweet = status.retweeted
tweet.retweet_count = status.retweet_count
tweet.is_quote = status.is_quote_status
if status.in_reply_to_status_id is not None:
tweet.is_reply = True
for media in status.entities.get("media", [{}]):
if media.get("type", None):
tweet.media_url = media.get('media_url')
if 'video' in tweet.media_url:
tweet.is_video = True
else:
tweet.is_image = True
# tweets.append(tweet)
tweet.save()
profile.tweets.add(tweet)
# profile.tweets.bulk_create(tweets)
profile.save()
def storeTimeline():
while 1:
try:
logger.info("Starting to store timeline data (pid %s)" % os.getpid())
twitter_stream = TwitterStream(domain="userstream.twitter.com")
db.init_db()
for msg in twitter_stream.stream.user(following=True):
if msg.get("text"):
tweet = Tweet()
tweet.text = msg["text"].encode("utf-8")
tweet.created_at = parser.parse(msg["created_at"]).replace(tzinfo=None)
if msg.get("coordinates"):
tweet.lon = msg["coordinates"]["coordinates"][0]
tweet.lat = msg["coordinates"]["coordinates"][1]
tweet.tweet_id = msg["id"]
tweet.retweet_count = msg["retweet_count"]
tweet.user_id = msg["user"]["id"]
db.session.add(tweet)
db.session.commit()
logger.error("Stream timeout or other cause for shutdown")
except Exception, err:
logger.error("%s, %s" % (Exception, err))
raise
def _handle_tweets(self, user):
try:
tweets = self.api.user_timeline(user.screen_name, count=200)
except tweepy.error.TweepError:
print('Error downloading tweets for {}'.format(user.screen_name))
return
for tweet_data in tweets:
tweet = self.session.query(Tweet).get(tweet_data.id)
if tweet is None:
tweet = Tweet()
is_new = True
else:
is_new = False
tweet.id = tweet_data.id
tweet.text = tweet_data.text
tweet.retweet_count = tweet_data.retweet_count
tweet.user = user
self.session.add(tweet)
self.session.commit()
if is_new:
print('Saving entities for tweet {}'.format(tweet.id))
self._handle_entities(tweet, tweet_data)
self.session.commit()
else:
print('Skipping entities for tweet {}'.format(tweet.id))
print('Downloaded {} tweets for user {}'.format(
len(tweets), user.screen_name))
self.session.commit()
