def handle_data(data_str): """Handle data from the Twitter Streaming API, via the redis queue.""" # XXX debug only really. sys.stderr.write('.') # Decode into a unicode string. text = unicode(data_str, 'utf-8') # Try to parse the JSON text into a data dict. try: data = json.loads(text) except Exception as err: return logger.warn(err) # In a transaction. with transaction.manager: # If we're dealing with a status. if data.has_key('in_reply_to_status_id'): return handle_status(data, text) # If we're dealing with a deletion record. if data.has_key('delete'): return handle_deletion(data) # XXX more events, e.g.: handle verification. # Close the db connection. Session.remove()
def setUp(self): from balistos import configure Session.remove() createTestDB() self.config = testing.setUp() configure(self.config) app = self.config.make_wsgi_app() from webtest import TestApp self.testapp = TestApp(app)
def main(args=None): """Consume the Twitter Streaming API.""" # Write a pid file. f = open('stream.pid', 'w') f.write(str(os.getpid())) f.close() # Parse the command line args. if args is None: args = parse_args() # Read the config file. config = ConfigParser.SafeConfigParser() config.read(args.config_file) # Setup logging. logging.config.fileConfig(args.config_file) # Patch sockets and threading. from gevent import monkey monkey.patch_all() import gevent_psycopg2 gevent_psycopg2.monkey_patch() # Bind the model classes. engine = create_engine(config.get('app:beliveat', 'sqlalchemy.url')) bind_engine(engine) # Instantiate a ``Manager`` with a redis client and oauth handler and # start the manager running. client = get_redis_client() handler = oauth_handler_factory(config) manager = Manager(client, handler, args.input_channel, args.output_channel) # Close the db connection Session.remove() try: manager.start() except KeyboardInterrupt: manager.stop()
def on_join(self, msg): """""" # Subscribe to redis notifications, listening for any events matching # ``*.user.canonical_id``. user = self.request.user pattern = '*:{0}'.format(user.canonical_id) redis = get_redis_client() subscriber = redis.pubsub() subscriber.psubscribe([pattern]) logger.debug(u'Subscribing {0} to redis notifications'.format(user.username)) # Close the db session. Session.remove() for notification in subscriber.listen(): parts = notification['channel'].split(':') name = parts[0] hashtag = parts[1] self.emit('notification', name, hashtag, notification['data'])
def main(args=None): """Process the ``INPUT_CHANNEL`` redis queue.""" # Write a pid file. f = open('queue.pid', 'w') f.write(str(os.getpid())) f.close() # Parse the command line args. if args is None: args = parse_args() # Read the config file. config = ConfigParser.SafeConfigParser() config.read(args.config_file) # Setup logging. logging.config.fileConfig(args.config_file) # Patch sockets, threading and the db driver. from gevent import monkey monkey.patch_all() import gevent_psycopg2 gevent_psycopg2.monkey_patch() # Bind the model classes. engine = create_engine(config.get('app:beliveat', 'sqlalchemy.url')) bind_engine(engine) # Setup the redis queue processor. client = get_redis_client() processor = QueueProcessor(client, [args.input_channel], handle_data) # Close the db connection Session.remove() try: processor.start() except KeyboardInterrupt: pass
def reload_predicates(self, get_twitter_ids=None, get_keywords=None): """Load the filter predicates.""" logger.warn('Reloading predicates...') logger.warn('- current:') logger.warn(self.follow_ids) logger.warn(self.track_keywords) if get_twitter_ids is None: get_twitter_ids = get_all_twitter_ids if get_keywords is None: get_keywords = get_track_keywords self.follow_ids = get_twitter_ids() self.track_keywords = get_keywords() # Close the db connection Session.remove() logger.warn('- new:') logger.warn(self.follow_ids) logger.warn(self.track_keywords)
def tearDown(self): Session.remove() testing.tearDown()
def tearDown(self): Session.remove() testing.tearDown()
def tearDown(self): """Make sure the session is cleared between tests.""" Session.remove()
def tearDown(self): """Make sure the session is cleared between tests.""" Session.remove()
def tearDown(self): handler.clear() Session.remove() testing.tearDown()
def tearDown(self): """Make sure the session is cleared between tests.""" from pyramid_basemodel import Session Session.remove()