def main():
connection = kombu.BrokerConnection(**config.get_rabbitmq_config())
channel = connection.channel()
twitter_exchange = kombu.Exchange("twitter", type="topic")
producer = kombu.Producer(channel, twitter_exchange)
control_queue = connection.SimpleBuffer("twitter_pump")
try:
twitter = twython.Twython(**config.get_twitter_config())
handle_tweets(twitter, producer, control_queue)
finally:
channel.close()
connection.close()
import pymongo
import sys
from twisted.internet import task
from twisted.internet import reactor
from twisted.python import log
import twython
from utils import config, urls
log.startLogging(sys.stdout)
api = twython.Twython(**config.get_twitter_config())
def get_since_id(coll, user_name):
since_id = coll.find_one({"user_name": user_name})
return since_id if since_id else dict(user_name=user_name, id=0)
def pull_and_save_tweets(api):
cfg = config.get("db")
uri = "%s:%s@%s/%s" % (
cfg.get("user"), cfg.get("password"), cfg.get("host"), cfg.get("db"))
conn = pymongo.Connection(uri)
db = conn[cfg.get("db")]
messages = db[cfg.get("messages_coll")]
messages.ensure_index("id")