Exemplo n.º 1
0
def get_tweets(location):
    geocode = '%s,%s,%s' % (location[0], location[1], SEARCH_RADIUS)
    cache_key = 'twitter%s' % geocode
    search = cache.get(cache_key)
    if not search:
        try:
            # Ugly but we need to provide search query in different languages...
            # TODO: implement diffrent appproach after adding Twitter signup.
            query = Search.objects.all()[0].query
        except IndexError:
            query = 'bike OR bicycle'

        search = api.search(query, geocode=geocode)
        cache.set(cache_key, search)
    return search
Exemplo n.º 2
0
 def test_save_tweets(self):
     tweets = api.search(lang='zh', q='*')
     for t in tweets:
         print t.text
         Tweet.objects.create_tweet(t)
     self.assertTrue(Tweet.objects.count() > 10)
Exemplo n.º 3
0
# If results from a specific ID onwards are reqd, set since_id to that ID.
# else default to no lower limit, go as far back as API allows
sinceId = None

# If results only below a specific ID are, set max_id to that ID.
# else default to no upper limit, start from the most recent tweet matching the search query.
max_id = -1L

tweetCount = 0
print("Downloading max {0} tweets".format(maxTweets))
with open(fName, 'w') as f:
    while tweetCount < maxTweets:
        try:
            if (max_id <= 0):
                if (not sinceId):
                    new_tweets = api.search(q=searchQuery, count=tweetsPerQry)
                else:
                    new_tweets = api.search(q=searchQuery,
                                            count=tweetsPerQry,
                                            since_id=sinceId)
            else:
                if (not sinceId):
                    new_tweets = api.search(q=searchQuery,
                                            count=tweetsPerQry,
                                            max_id=str(max_id - 1))
                else:
                    new_tweets = api.search(q=searchQuery,
                                            count=tweetsPerQry,
                                            max_id=str(max_id - 1),
                                            since_id=sinceId)
            if not new_tweets:
Exemplo n.º 4
0
        lock_file = '/tmp/tengtweets_fetch_lock.tmp'
        if os.path.exists(lock_file):
            exit(0)

        from tengtweets.tweets.models import Tweet
        from tweepy import api
        import logging
        import time

        open(lock_file, 'w').close()
        logger = logging.getLogger('info')
        logger_error = logging.getLogger('error')
        count = 0
        count_zh = 0
        for i in range(5):
            tweets = api.search(lang='zh', q='*', rpp=100)
            for t in tweets:
                try:
                    if (Tweet.objects.create_tweet(t)):
                        count += 1
                        count_zh += 1
                except Exception, e:
                    logger_error.error("Error: %s id: %s text: %s" % (
                        e, t.id, t.text
                        ))
            time.sleep(8)

            tweets = api.search(lang='en', q='*', rpp=10)
            for t in tweets:
                try:
                    if (Tweet.objects.create_tweet(t)):
# If results from a specific ID onwards are reqd, set since_id to that ID.
# else default to no lower limit, go as far back as API allows
sinceId = None

# If results only below a specific ID are, set max_id to that ID.
# else default to no upper limit, start from the most recent tweet matching the search query.
max_id = -1L

tweetCount = 0
print("Downloading max {0} tweets".format(maxTweets))
with open(fName, "w") as f:
    while tweetCount < maxTweets:
        try:
            if max_id <= 0:
                if not sinceId:
                    new_tweets = api.search(q=searchQuery, count=tweetsPerQry)
                else:
                    new_tweets = api.search(q=searchQuery, count=tweetsPerQry, since_id=sinceId)
            else:
                if not sinceId:
                    new_tweets = api.search(q=searchQuery, count=tweetsPerQry, max_id=str(max_id - 1))
                else:
                    new_tweets = api.search(q=searchQuery, count=tweetsPerQry, max_id=str(max_id - 1), since_id=sinceId)
            if not new_tweets:
                print("No more tweets found")
                break
            for tweet in new_tweets:
                f.write(jsonpickle.encode(tweet._json, unpicklable=False) + "\n")
            tweetCount += len(new_tweets)
            print("Downloaded {0} tweets".format(tweetCount))
            max_id = new_tweets[-1].id