def app_main(self, config, options, args):
# TODO: deal with new user registrations by listening to amqp and schedule the rest
observer = PythonLoggingObserver()
observer.start()
# initialize some important maps
db.configure_session(db.create_url_from_config(config['db']))
service.initialize()
post_type.initialize()
# Grab twitter consumer keys
self.consumer_key = config['oauth']['twitter']['key']
self.consumer_secret = config['oauth']['twitter']['secret']
self.default_token_key = config['oauth']['twitter']['default_access_token']
self.default_token_secret = config['oauth']['twitter']['default_access_token_secret']
# Grab feed configuration
self.wait_on_collector_query_delay = float(config['feed']['wait_on_collector_query_delay'])
# Configure amqp
amqp_host = config['amqp']['broker']['host']
amqp_port = int(config['amqp']['broker']['port'])
amqp_spec = message_queue.create_spec_path(config['amqp']['broker']['spec'])
self.amqp_exchange = config['amqp']['exchange']['name']
self.amqp = AmqpFactory(host=amqp_host, port=amqp_port, spec_file=amqp_spec)
db_host = config['db']['host']
db_user = config['db']['user']
db_passwd = config['db']['password']
db_database = config['db']['database']
db_unicode = to_bool(config['db']['unicode'])
self.db_pool = adbapi.ConnectionPool(
'MySQLdb',
host=db_host,
user=db_user,
passwd=db_passwd,
db=db_database,
use_unicode=db_unicode,
cp_noisy=True)
self._process_twitter_users()
reactor.run()
class EventTwitterFeedDriver(app_base.AppBase):
user_ids = set()
def app_main(self, config, options, args):
# TODO: deal with new user registrations by listening to amqp and schedule the rest
observer = PythonLoggingObserver()
observer.start()
# initialize some important maps
db.configure_session(db.create_url_from_config(config['db']))
service.initialize()
post_type.initialize()
# Grab twitter consumer keys
self.consumer_key = config['oauth']['twitter']['key']
self.consumer_secret = config['oauth']['twitter']['secret']
self.default_token_key = config['oauth']['twitter']['default_access_token']
self.default_token_secret = config['oauth']['twitter']['default_access_token_secret']
# Grab feed configuration
self.wait_on_collector_query_delay = float(config['feed']['wait_on_collector_query_delay'])
# Configure amqp
amqp_host = config['amqp']['broker']['host']
amqp_port = int(config['amqp']['broker']['port'])
amqp_spec = message_queue.create_spec_path(config['amqp']['broker']['spec'])
self.amqp_exchange = config['amqp']['exchange']['name']
self.amqp = AmqpFactory(host=amqp_host, port=amqp_port, spec_file=amqp_spec)
db_host = config['db']['host']
db_user = config['db']['user']
db_passwd = config['db']['password']
db_database = config['db']['database']
db_unicode = to_bool(config['db']['unicode'])
self.db_pool = adbapi.ConnectionPool(
'MySQLdb',
host=db_host,
user=db_user,
passwd=db_passwd,
db=db_database,
use_unicode=db_unicode,
cp_noisy=True)
self._process_twitter_users()
reactor.run()
def _handle_twitter_query_result(self, result_set):
users_without_oauth = []
for twitter_user in result_set:
# start a connection to twitter
asm = _create_author_service_map(twitter_user)
if (asm.access_token and
asm.access_token_secret):
self._handle_users(asm.access_token, asm.access_token_secret, [asm])
else:
users_without_oauth.append(asm)
if users_without_oauth:
self._handle_users(self.default_token_key, self.default_token_secret, users_without_oauth)
def _handle_users(self, token_key, token_secret, author_service_maps):
handlers = []
for asm in author_service_maps:
if (asm.service_author_id not in self.user_ids):
self.user_ids.add(asm.service_author_id)
handler = TwitterHandler(asm, self)
handlers.append(handler)
group_handler = GroupTwitterHandler(handlers, self, token_key, token_secret)
group_handler.start_feed()
def notify_event_collector(self, asm):
notification = messages.create_notification_message('twitter', asm.service_author_id)
routing_key = notification['header']['type']
body = json_serializer.dump_string(notification)
self.amqp.send_message(
exchange=self.amqp_exchange,
routing_key=routing_key,
msg=body)
def listen_to_twitter(self, token_key, token_secret, handler):
method = 'POST'
url = 'https://stream.twitter.com/1/statuses/filter.json'
data = {'follow': ','.join(handler.list_of_twitter_ids())}
headers = _sign_request(
method=method,
url=url,
data=data,
consumer_key=self.consumer_key,
consumer_secret=self.consumer_secret,
token_key=token_key,
token_secret=token_secret)
headers['Authorization'] = [headers['Authorization'].encode('ascii')]
headers['Content-Type'] = ['application/x-www-form-urlencoded']
body = FileBodyProducer(StringIO(urllib.urlencode(data)))
contextFactory = WebClientContextFactory()
agent = Agent(reactor, contextFactory)
d = agent.request(
method,
url,
Headers(headers),
body)
def handle_response(response):
if response.code == 200:
logging.info('Starting to listen for stream from: %s', data)
response.deliverBody(TwitterStreamProtocol(handler))
handler.connected()
elif response.code == 401:
logging.info('Authorization failed. Try to reconnect: %s', data)
handler.reconnect_feed()
else:
logging.error(
'Got a bad response: %s, phrase: %s, for: %s',
response.code,
response.phrase,
data)
def handle_connection_refused(failure):
failure.trap(twisted.internet.error.ConnectionRefusedError)
logging.warning('Connection refused for: %s', data)
handler.reconnect_feed()
d.addCallbacks(handle_response, handle_connection_refused)
d.addErrback(err)
def _process_twitter_users(self):
twitter_service_id = 2 # TODO: would be nice to remove this
deferred = self.db_pool.runQuery(
'''SELECT author_id, service_id, access_token, access_token_secret, service_author_id,
id, last_update_time, most_recent_event_id, most_recent_event_timestamp
FROM author_service_map
WHERE service_id = %s;''',
(twitter_service_id,))
deferred.addCallback(self._handle_twitter_query_result)
deferred.addErrback(err)