def load_tweets(profile, api): tweets = [] for status in tweepy.Cursor(api.user_timeline).items(): print str(status.id) tweet = Tweet(temp=str(status.id)[-9:]) tweet.text = unicode(status.text) tweet.created = timezone.make_aware(status.created_at, timezone.get_current_timezone()) tweet.is_retweet = status.retweeted tweet.retweet_count = status.retweet_count tweet.is_quote = status.is_quote_status if status.in_reply_to_status_id is not None: tweet.is_reply = True for media in status.entities.get("media", [{}]): if media.get("type", None): tweet.media_url = media.get('media_url') if 'video' in tweet.media_url: tweet.is_video = True else: tweet.is_image = True # tweets.append(tweet) tweet.save() profile.tweets.add(tweet) # profile.tweets.bulk_create(tweets) profile.save()
def storeTimeline(): while 1: try: logger.info("Starting to store timeline data (pid %s)" % os.getpid()) twitter_stream = TwitterStream(domain="userstream.twitter.com") db.init_db() for msg in twitter_stream.stream.user(following=True): if msg.get("text"): tweet = Tweet() tweet.text = msg["text"].encode("utf-8") tweet.created_at = parser.parse(msg["created_at"]).replace(tzinfo=None) if msg.get("coordinates"): tweet.lon = msg["coordinates"]["coordinates"][0] tweet.lat = msg["coordinates"]["coordinates"][1] tweet.tweet_id = msg["id"] tweet.retweet_count = msg["retweet_count"] tweet.user_id = msg["user"]["id"] db.session.add(tweet) db.session.commit() logger.error("Stream timeout or other cause for shutdown") except Exception, err: logger.error("%s, %s" % (Exception, err)) raise
def _handle_tweets(self, user): try: tweets = self.api.user_timeline(user.screen_name, count=200) except tweepy.error.TweepError: print('Error downloading tweets for {}'.format(user.screen_name)) return for tweet_data in tweets: tweet = self.session.query(Tweet).get(tweet_data.id) if tweet is None: tweet = Tweet() is_new = True else: is_new = False tweet.id = tweet_data.id tweet.text = tweet_data.text tweet.retweet_count = tweet_data.retweet_count tweet.user = user self.session.add(tweet) self.session.commit() if is_new: print('Saving entities for tweet {}'.format(tweet.id)) self._handle_entities(tweet, tweet_data) self.session.commit() else: print('Skipping entities for tweet {}'.format(tweet.id)) print('Downloaded {} tweets for user {}'.format( len(tweets), user.screen_name)) self.session.commit()