def get_tweets(location): geocode = '%s,%s,%s' % (location[0], location[1], SEARCH_RADIUS) cache_key = 'twitter%s' % geocode search = cache.get(cache_key) if not search: try: # Ugly but we need to provide search query in different languages... # TODO: implement diffrent appproach after adding Twitter signup. query = Search.objects.all()[0].query except IndexError: query = 'bike OR bicycle' search = api.search(query, geocode=geocode) cache.set(cache_key, search) return search
def test_save_tweets(self): tweets = api.search(lang='zh', q='*') for t in tweets: print t.text Tweet.objects.create_tweet(t) self.assertTrue(Tweet.objects.count() > 10)
# If results from a specific ID onwards are reqd, set since_id to that ID. # else default to no lower limit, go as far back as API allows sinceId = None # If results only below a specific ID are, set max_id to that ID. # else default to no upper limit, start from the most recent tweet matching the search query. max_id = -1L tweetCount = 0 print("Downloading max {0} tweets".format(maxTweets)) with open(fName, 'w') as f: while tweetCount < maxTweets: try: if (max_id <= 0): if (not sinceId): new_tweets = api.search(q=searchQuery, count=tweetsPerQry) else: new_tweets = api.search(q=searchQuery, count=tweetsPerQry, since_id=sinceId) else: if (not sinceId): new_tweets = api.search(q=searchQuery, count=tweetsPerQry, max_id=str(max_id - 1)) else: new_tweets = api.search(q=searchQuery, count=tweetsPerQry, max_id=str(max_id - 1), since_id=sinceId) if not new_tweets:
lock_file = '/tmp/tengtweets_fetch_lock.tmp' if os.path.exists(lock_file): exit(0) from tengtweets.tweets.models import Tweet from tweepy import api import logging import time open(lock_file, 'w').close() logger = logging.getLogger('info') logger_error = logging.getLogger('error') count = 0 count_zh = 0 for i in range(5): tweets = api.search(lang='zh', q='*', rpp=100) for t in tweets: try: if (Tweet.objects.create_tweet(t)): count += 1 count_zh += 1 except Exception, e: logger_error.error("Error: %s id: %s text: %s" % ( e, t.id, t.text )) time.sleep(8) tweets = api.search(lang='en', q='*', rpp=10) for t in tweets: try: if (Tweet.objects.create_tweet(t)):
# If results from a specific ID onwards are reqd, set since_id to that ID. # else default to no lower limit, go as far back as API allows sinceId = None # If results only below a specific ID are, set max_id to that ID. # else default to no upper limit, start from the most recent tweet matching the search query. max_id = -1L tweetCount = 0 print("Downloading max {0} tweets".format(maxTweets)) with open(fName, "w") as f: while tweetCount < maxTweets: try: if max_id <= 0: if not sinceId: new_tweets = api.search(q=searchQuery, count=tweetsPerQry) else: new_tweets = api.search(q=searchQuery, count=tweetsPerQry, since_id=sinceId) else: if not sinceId: new_tweets = api.search(q=searchQuery, count=tweetsPerQry, max_id=str(max_id - 1)) else: new_tweets = api.search(q=searchQuery, count=tweetsPerQry, max_id=str(max_id - 1), since_id=sinceId) if not new_tweets: print("No more tweets found") break for tweet in new_tweets: f.write(jsonpickle.encode(tweet._json, unpicklable=False) + "\n") tweetCount += len(new_tweets) print("Downloaded {0} tweets".format(tweetCount)) max_id = new_tweets[-1].id