def handle_data(data_str):
"""Handle data from the Twitter Streaming API, via the redis queue."""
# XXX debug only really.
sys.stderr.write('.')
# Decode into a unicode string.
text = unicode(data_str, 'utf-8')
# Try to parse the JSON text into a data dict.
try:
data = json.loads(text)
except Exception as err:
return logger.warn(err)
# In a transaction.
with transaction.manager:
# If we're dealing with a status.
if data.has_key('in_reply_to_status_id'):
return handle_status(data, text)
# If we're dealing with a deletion record.
if data.has_key('delete'):
return handle_deletion(data)
# XXX more events, e.g.: handle verification.
# Close the db connection.
Session.remove()
def setUp(self):
from balistos import configure
Session.remove()
createTestDB()
self.config = testing.setUp()
configure(self.config)
app = self.config.make_wsgi_app()
from webtest import TestApp
self.testapp = TestApp(app)
def main(args=None):
"""Consume the Twitter Streaming API."""
# Write a pid file.
f = open('stream.pid', 'w')
f.write(str(os.getpid()))
f.close()
# Parse the command line args.
if args is None:
args = parse_args()
# Read the config file.
config = ConfigParser.SafeConfigParser()
config.read(args.config_file)
# Setup logging.
logging.config.fileConfig(args.config_file)
# Patch sockets and threading.
from gevent import monkey
monkey.patch_all()
import gevent_psycopg2
gevent_psycopg2.monkey_patch()
# Bind the model classes.
engine = create_engine(config.get('app:beliveat', 'sqlalchemy.url'))
bind_engine(engine)
# Instantiate a ``Manager`` with a redis client and oauth handler and
# start the manager running.
client = get_redis_client()
handler = oauth_handler_factory(config)
manager = Manager(client, handler, args.input_channel, args.output_channel)
# Close the db connection
Session.remove()
try:
manager.start()
except KeyboardInterrupt:
manager.stop()
def on_join(self, msg):
""""""
# Subscribe to redis notifications, listening for any events matching
# ``*.user.canonical_id``.
user = self.request.user
pattern = '*:{0}'.format(user.canonical_id)
redis = get_redis_client()
subscriber = redis.pubsub()
subscriber.psubscribe([pattern])
logger.debug(u'Subscribing {0} to redis notifications'.format(user.username))
# Close the db session.
Session.remove()
for notification in subscriber.listen():
parts = notification['channel'].split(':')
name = parts[0]
hashtag = parts[1]
self.emit('notification', name, hashtag, notification['data'])
def main(args=None):
"""Process the ``INPUT_CHANNEL`` redis queue."""
# Write a pid file.
f = open('queue.pid', 'w')
f.write(str(os.getpid()))
f.close()
# Parse the command line args.
if args is None:
args = parse_args()
# Read the config file.
config = ConfigParser.SafeConfigParser()
config.read(args.config_file)
# Setup logging.
logging.config.fileConfig(args.config_file)
# Patch sockets, threading and the db driver.
from gevent import monkey
monkey.patch_all()
import gevent_psycopg2
gevent_psycopg2.monkey_patch()
# Bind the model classes.
engine = create_engine(config.get('app:beliveat', 'sqlalchemy.url'))
bind_engine(engine)
# Setup the redis queue processor.
client = get_redis_client()
processor = QueueProcessor(client, [args.input_channel], handle_data)
# Close the db connection
Session.remove()
try:
processor.start()
except KeyboardInterrupt:
pass
def reload_predicates(self, get_twitter_ids=None, get_keywords=None):
"""Load the filter predicates."""
logger.warn('Reloading predicates...')
logger.warn('- current:')
logger.warn(self.follow_ids)
logger.warn(self.track_keywords)
if get_twitter_ids is None:
get_twitter_ids = get_all_twitter_ids
if get_keywords is None:
get_keywords = get_track_keywords
self.follow_ids = get_twitter_ids()
self.track_keywords = get_keywords()
# Close the db connection
Session.remove()
logger.warn('- new:')
logger.warn(self.follow_ids)
logger.warn(self.track_keywords)
def tearDown(self):
Session.remove()
testing.tearDown()
def tearDown(self):
Session.remove()
testing.tearDown()
def tearDown(self):
"""Make sure the session is cleared between tests."""
Session.remove()
def tearDown(self):
"""Make sure the session is cleared between tests."""
Session.remove()
def tearDown(self):
handler.clear()
Session.remove()
testing.tearDown()
def tearDown(self):
"""Make sure the session is cleared between tests."""
from pyramid_basemodel import Session
Session.remove()
