def _get_streaming_args(self):
self._credentials = settings.Twitter['accounts'][self._daemon_number]
segs = ["C" + code for code in Geo.country_codes()]
#We're only allowed to track 400 keywords
lock = FileLock('/tmp/trackwords')
with lock:
all_hashtags = self._get_all_entities("hashtag", segs)
all_usermentions = self._get_all_entities("user_mention", segs)
used_hashtags, used_usermentions = map(set, self._get_used_trackwords())
hashtags = [ht for ht in all_hashtags if not ht in used_hashtags][:200]
usermentions = [um for um in all_usermentions if not um in used_usermentions][:200]
self._set_used_trackwords(hashtags, usermentions)
self._payload = {'track': hashtags + usermentions}
return self._credentials, self._payload
def cache_top_tweets():
#initialize stores
ts = int(time.time())
countstore = CountStore()
tweetstore = TweetStore()
cache = RedisCache(namespace=settings.TopTweetsCache["namespace"])
countries = Geo.country_codes()
top_tweets_cache = {}
for country in countries:
print "*************"
print country
print "*************"
top_tweets = {}
segmentation = "C" + country
for entitytype in ["hashtag", "user_mention"]:
top_tweets[entitytype] = []
top_entities = countstore.get_top(entitytype,
segmentation,
settings.Aggregation['top_entities'],
ts)
for entity, count in top_entities:
data = {"text":entity, "count":count, "tweets":[]}
tweets = top_tweets_cache.get((entitytype, entity, ts))
if not tweets:
print "fetching tweets for " + str((entitytype, entity, ts))
segmentation = ":".join([entitytype, entity])
tweets = countstore.get_top("tweet",
segmentation,
settings.Aggregation['top_tweets'],
ts)
tweets = map(lambda x: (tweetstore.get(x[0]), x[1]), tweets)
top_tweets_cache[(entitytype, entity, ts)] = tweets
for tweet, count in tweets:
data["tweets"].append({"tweet":tweet.data, "count": count})
top_tweets[entitytype].append(data)
cache.put(segmentation, top_tweets)
