class QueuePublisherClient(object):
def __init__(self, queue, request):
self.queue = queue
self.response = None
self.channel = None
self.request = request
self.corrId = str(uuid.uuid4())
self.callBackQueue = None
self.connection = None
parameters = pika.ConnectionParameters(host="0.0.0.0")
self.connection = SelectConnection(
parameters, self.on_response_connected
)
self.connection.ioloop.start()
def on_response(self,ch, method, props, body):
if self.corrId == props.correlation_id:
self.response = body
self.connection.close()
self.connection.ioloop.start()
def on_response_connected(self,connection):
_logger.info("Connected...\t(%s)" % self.queue)
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_response_channel_open(self,channel):
self.responseChannel = channel
result = self.responseChannel.queue_declare(
exclusive=True, callback=self.on_response_queue_declared
)
def on_connected(self,connection):
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
_logger.info("Channel Opened...\t(%s)" % self.queue)
self.channel = channel
self.channel.queue_declare(queue = self.queue,
durable=True,
exclusive=False,
auto_delete=False,
callback=self.on_queue_declared)
def on_queue_declared(self,frame):
self.channel.basic_publish(exchange="",
routing_key = self.queue,
properties = pika.BasicProperties(),
body=str(self.request))
self.connection.close()
_logger.info("Message Published...\t(%s)" % self.queue)
class DataProvisionClient(object):
def __init__(self, queue, request):
self.queue = queue
self.response = None
self.channel = None
self.request = request
self.corrId = str(uuid.uuid4())
self.callBackQueue = None
self.connection = None
parameters = pika.ConnectionParameters(host='127.0.0.1')
self.connection = SelectConnection(parameters,
self.on_response_connected)
self.connection.ioloop.start()
def on_response(self, ch, method, props, body):
if self.corrId == props.correlation_id:
self.response = body
self.connection.close()
self.connection.ioloop.start()
def on_response_connected(self, connection):
print('connected ..')
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_response_channel_open(self, channel):
self.responseChannel = channel
result = self.responseChannel.queue_declare(
exclusive=True, callback=self.on_response_queue_declared)
def on_connected(self, connection):
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
print('channel opened')
self.channel = channel
self.channel.queue_declare(queue=self.queue,
durable=True,
exclusive=False,
auto_delete=False,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.basic_publish(exchange='',
routing_key=self.queue,
properties=pika.BasicProperties(),
body=str(self.request))
self.connection.close()
print('message delivered')
class NewsFeedBee(Thread):
def __init__(self, options):
super(NewsFeedBee, self).__init__()
self.options = options
self.dispatcher = None
self.feeds = []
self.update_feeds()
# RabbitMQ stuffs
self.channel = None
self.connection = None
self.alive = False
self.stop_flag = False
#self.r = None
#self.r_last_touched = datetime.now()
def update_feeds(self):
global db_conn
try:
LOGGER.info("Obtaining feeds to process from database")
if db_conn is None:
connect_pg()
cursor = db_conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cursor.execute("SELECT id, name, feed_url, etag, last_modified " \
"FROM feedjack_feed " \
"WHERE is_active=%s "
"ORDER BY id", (True,))
results = cursor.fetchall()
for result in results:
#if result['last_modified'] == None:
# last_modified = None
#else:
# last_modified = str(result['last_modified'])
self.feeds.append({
'id': result['id'],
'name': result['name'],
'feed_url': str(result['feed_url']),
'etag': str(result['etag']),
'last_modified': result['last_modified']
})
LOGGER.info("Obtaining feeds to process from database completed")
except:
db_conn.rollback()
LOGGER.info(sys.exc_info())
def post_init(self):
# 初始化分发任务类dispatcher
# 通过异步的方式 打开rabbitMQ的连接，并新建channel，声明exchange，声明queue，绑定exchange和queue，然后运行run方法，之前运行ioloop
self.dispatcher = Dispatcher(self.options)
try:
LOGGER.info('Opening a pika connection')
self.connection = SelectConnection(
parameters=settings.pika_parameters,
on_open_callback=self.on_connection_open,
on_open_error_callback=self.on_connection_open_error)
try:
LOGGER.info('Starting ioloop')
self.connection.ioloop.start()
except KeyboardInterrupt:
# Gracefully close the connection
self.connection.close()
# Loop until we're fully closed, will stop on its own
self.connection.ioloop.start()
except:
(etype, eobj, etb) = sys.exc_info()
print traceback.format_exception(etype, eobj, etb)
def on_connection_open(self, unused_connection):
LOGGER.info('Opening a connection completed')
self.open_channel()
def on_connection_open_error(self, connection):
LOGGER.info('Opening a connection failed')
def open_channel(self):
LOGGER.info('Opening a channel')
self.connection.channel(on_open_callback=self.on_channel_open)
def on_channel_open(self, new_channel):
LOGGER.info('Opening a channel completed')
self.channel = new_channel
self.declare_exchange()
def declare_exchange(self):
LOGGER.info('Declaring an exchange')
self.channel.exchange_declare(
exchange=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_NAME,
exchange_type=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_TYPE,
passive=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_PASSIVE,
durable=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_DURABLE,
auto_delete=settings.
RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_AUTO_DELETE,
internal=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_INTERNAL,
nowait=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_NOWAIT,
arguments=None, # Custom key/value pair arguments for the exchange
callback=self.on_exchange_declared
) # Call this method on Exchange.DeclareOk
def on_exchange_declared(self, frame):
LOGGER.info('Declaring an exchange completed')
self.declare_queue()
def declare_queue(self):
LOGGER.info('Declaring a queue')
self.channel.queue_declare(
self.on_queue_declared,
settings.RABBITMQ_NEWSFEED_RAW_FEED_QUEUE_NAME)
def on_queue_declared(self, method_frame):
LOGGER.info('Declaring a queue completed')
self.bind_queue()
def bind_queue(self):
LOGGER.info('Binding a queue')
self.channel.queue_bind(
callback=self.on_queue_binded,
queue=settings.RABBITMQ_NEWSFEED_RAW_FEED_QUEUE_NAME,
exchange=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_NAME,
routing_key=settings.RABBITMQ_NEWSFEED_RAW_FEED_ROUTING_KEY)
def on_queue_binded(self, frame):
LOGGER.info('Binding a queue completed')
self.start() # Start the thread's activity
def run(self):
while True:
for feed in self.feeds:
if not self.dispatcher:
self.dispatcher = Dispatcher(self.options)
#if not self.channel:
# self.on_connected()
#返回在dispatcher处理后传来的数据（即是feedparser抓取的数据）
data = self.dispatcher.add_job(feed)
if data is not None and isinstance(data, dict):
if settings.DEBUG:
filename = 'tmp/feed_' + str(feed['id']) + '_raw.txt'
with open(filename, 'w') as f:
f.write((pformat(data)).decode('utf-8'))
if hasattr(data, 'updated_parsed'):
del data['updated_parsed']
if hasattr(data['feed'], 'published_parsed'):
#data['feed']['published'] = datetime.fromtimestamp(
# mktime(data['feed']['published_parsed']))
del data['feed']['published_parsed']
if hasattr(data['feed'], 'updated_parsed'):
#data['feed']['updated'] = datetime.fromtimestamp(
# mktime(data['feed']['updated_parsed']))
del data['feed']['updated_parsed']
# "data['entries']" is a list of dictionaries.
# Each dictionary contains data from a different entry.
for idx, val in enumerate(data['entries']):
if hasattr(val, 'created_parsed'):
#data['entries'][idx]['created'] = datetime.fromtimestamp(
# mktime(val['created_parsed']))
del data['entries'][idx]['created_parsed']
if hasattr(val, 'expired_parsed'):
del data['entries'][idx]['expired_parsed']
if hasattr(val, 'published_parsed'):
del data['entries'][idx]['published_parsed']
if hasattr(val, 'updated_parsed'):
del data['entries'][idx]['updated_parsed']
data['feed_id'] = feed['id']
#data['last_modified'] = feed['last_modified']
try:
# If this throws an error, don't put message in queue
json_data = JSON_ENCODER.encode(data)
if settings.DEBUG:
filename = 'tmp/feed_' + str(feed['id']) + '.json'
with open(filename, 'w') as f:
#simplejson.dump(json_data, f)
f.write(json_data)
LOGGER.info('Publishing data to queue')
self.channel.basic_publish(
exchange=settings.
RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_NAME, # The exchange to publish to
routing_key=settings.
RABBITMQ_NEWSFEED_RAW_FEED_ROUTING_KEY, # The routing key to bind on
body=json_data) # The message body
LOGGER.info('Publishing data to queue completed')
except simplejson.JSONDecodeError:
LOGGER.info(sys.exc_info())
except KeyboardInterrupt:
LOGGER.info("KeyboardInterrupt, so quitting! Bye!")
quit()
except SystemExit:
quit()
except:
LOGGER.info(sys.exc_info())
# Suspend execution for the given number of seconds.
LOGGER.info('Sleeping ' + str(self.options.sleeptime) +
' seconds. Zzz...')
sleep(self.options.sleeptime)
def stop(self):
pass
class dataChannel(object):
""" The dataChannel is the base class of all our datasource.
It's purpose is to: a). Setup the queues"""
def __init__(self, server_name='test', mq_exchange='', mq_queue = '',mq_host=''):
self.channel = None
self.id = server_name
self.queue_counter = 0
self.queue =mq_queue
self.routing_key = ''
self.exchange = mq_exchange
self.connection = None
self.connected = False
self.connecting = False
self.rabbithost = mq_host
logging.getLogger('pika').setLevel(logging.DEBUG)
def get_connection(self):
return self.connection
def connect(self):
if self.connecting:
return
self.connecting = True
credentials = pika.PlainCredentials('guest', 'guest')
params = pika.ConnectionParameters(host="hackinista.com",
port=5672,
virtual_host="/",
credentials=credentials)
host = (len(sys.argv) > 1) and sys.argv[1] or '127.0.0.1'
try:
self.connection = SelectConnection(params, self.on_connected)
except:
# self.L.critical("Error connecting to rabbitmq on host =
# "+self.host);
sys.exit(-1)
###
def on_connected(self, connection):
self.connection = connection
self.connection.channel(self.on_channel_open)
self.connected = True
def on_channel_open(self, channel):
self.channel = channel
try:
self.channel.queue_declare(queue=self.queue,
auto_delete=False,
durable=True,
exclusive=False,
callback=self.on_queue_declared)
except:
print "Error declaring queue = " + self.queue
sys.exit(-1)
def on_queue_declared(self, frame):
try:
self.channel.queue_bind(exchange=self.exchange,
queue=self.queue,
routing_key=self.routing_key,
callback=self.on_queue_bound)
except:
print "Binding to queue = " + self.queue
pass
def on_queue_bound(self, frame):
self.channel.basic_consume(consumer_callback=self.handle_delivery,
queue=self.queue, no_ack=False)
def handle_delivery(self, channel, method_frame, header_frame, body):
print "7...Basic.Deliver %s delivery-tag %i: %s" %\
(header_frame.content_type,
method_frame.delivery_tag,
body)
self.data_op(body)
channel.basic_ack(delivery_tag=method_frame.delivery_tag)
def data_op(self, args):
print "Please implement get_data"
class dataChannel:
""" The dataChannel is the base class of all our datasource.
It's purpose is to: a). Setup the queues"""
def __init__(self,ds_name):
self.channel = None
self.dc_id = "eek"#dc_id
## query mongoDB to find all the particulars about this
## data channel including: which queue is listening to,
## which exchange, the routing key..etc.
self.ret_queue = "ret_queue"
self.connection = None
self.channel = None
self.connected = False;
self.connecting = False;
self.exchange = "test_x";
self.queue = "test_q"
self.routing_key = "test_q"
## use the ds to the find which exchange and which queue this
## datachannel listens
def mongo_db(self):
## connect to the mongodb
mongo_conn = Connection('localhost',27017)
db = mongo_conn['data_channels']
coll= db['bbox_pts']
def connect(self):
print self
if self.connecting:
print ('1...PikaClient: Already connecting to RabbitMQ')
return
print ('1...PikaClient: Connecting to RabbitMQ on localhost:5672')
self.connecting = True
credentials = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(host='localhost',
port=5672,
virtual_host="/",
credentials=credentials)
host = (len(sys.argv) > 1) and sys.argv[1] or '127.0.0.1'
self.connection = SelectConnection(ConnectionParameters(host),
self.on_connected)
if self.connection != None:
print self.connection
print 'connection'
def on_connected(self,connection):
print '2...PikaClient: Connected to RabbitMQ on localhost:5672'
self.connection = connection
self.connection.channel(self.on_channel_open)
self.connected = True
def on_channel_open(self, channel):
print ('3...PikaClient: Channel Open, Declaring Exchange')
self.channel = channel
self.channel.exchange_declare(exchange=self.exchange,
type="direct",
auto_delete=False,
durable=True,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
print ('4...PikaClient: Exchange Declared, Declaring Queue')
self.channel.queue_declare(queue=self.queue,
auto_delete=False,
durable=True,
exclusive=False,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
print('5...PikaClient: Queue Declared, Binding Queue')
print "demo_receive: Queue Declared"
# self.channel.basic_consume(self.handle_delivery, queue='test_q')
self.channel.queue_bind(exchange=self.exchange,
queue=self.queue,
routing_key=self.routing_key,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
print('6...PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.handle_delivery,
queue=self.queue)
def handle_delivery(self,channel, method_frame, header_frame, body):
print "7...Basic.Deliver %s delivery-tag %i: %s" %\
(header_frame.content_type,
method_frame.delivery_tag,
body)
print body
channel.basic_ack(delivery_tag=method_frame.delivery_tag)
def get_data(self,args):
print "Please implement get_data"
class Amqp(object):
def __init__(self, conf):
# RabbitMQ general options
self.cacertfile = conf['cacertfile']
self.certfile = conf['certfile']
self.exchange = conf['exchange']
self.status_exchange = conf['status_exchange']
self.fail_if_no_peer_cert = conf['fail_if_no_peer_cert']
self.heartbeat = conf['heartbeat']
self.host = conf['host']
self.keyfile = conf['keyfile']
self.password = conf['password']
self.port = conf['port']
self.ssl_port = conf['ssl_port']
self.queue = conf['uuid']
self.retry_timeout = conf['retry_timeout']
self.ssl_auth = conf['ssl_auth']
self.use_ssl = conf['use_ssl']
self.username = conf['username']
self.vhost = conf['vhost']
# Connection and channel initialization
self.connection = None
self.channel = None
# Plain credentials
credentials = PlainCredentials(self.username, self.password)
pika_options = {'host': self.host,
'port': self.port,
'virtual_host': self.vhost,
'credentials': credentials}
# SSL options
if self.use_ssl:
pika_options['ssl'] = True
pika_options['port'] = self.ssl_port
if self.ssl_auth:
pika_options['credentials'] = ExternalCredentials()
pika_options['ssl_options'] = {
'ca_certs': self.cacertfile,
'certfile': self.certfile,
'keyfile': self.keyfile,
'cert_reqs': CERT_REQUIRED
}
if self.heartbeat:
pika_options['heartbeat'] = self.heartbeat
self.parameters = None
try:
self.parameters = pika.ConnectionParameters(**pika_options)
except TypeError as err:
self.logger.debug(err)
# Let's be compatible with original pika version (no integer for
# heartbeats and no ssl.
self.logger.warning("Wrong pika lib version, won't use ssl.")
pika_options['heartbeat'] = True
if self.use_ssl:
self.use_ssl = False
pika_options['port'] = self.port
del pika_options['ssl']
if self.ssl_auth:
self.ssl_auth = False
del pika_options['ssl_options']
self.parameters = pika.ConnectionParameters(**pika_options)
def connect(self):
SelectPoller.TIMEOUT = .1
self.connection = SelectConnection(self.parameters, self.on_connected)
self.connection.ioloop.start()
def close(self, amqperror=False):
if (self.connection and not self.connection.closing
and not self.connection.closed):
self.logger.debug("Closing connection")
self.connection.close()
#self.connection.ioloop.start()
def on_remote_close(self, code, text):
self.logger.debug("Remote channel close, code %d" % code)
time.sleep(2)
if code != 200:
self.close()
raise AmqpError(text)
def on_connection_closed(self, frame):
self.connection.ioloop.stop()
def on_connected(self, connection):
self.connection = connection
self.connection.add_on_close_callback(self.on_connection_closed)
self.connection.channel(self.on_channel_open)
class NewsFeedMaggot(Thread):
def __init__(self):
super(NewsFeedMaggot, self).__init__()
self.feeds = []
self.update_track_dict()
self.processfeed = ProcessFeed()
self.channel = None
self.connection = None
self.alive = False
self.stop_flag = False
self.r = None
self.r_last_touched = datetime.now()
def update_track_dict(self):
global DB_CONN
try:
LOGGER.info("Obtaining track from postgre")
if DB_CONN is None:
connect_pg()
# Get all ids from "feedjack_track"
track_ids = []
cursor = DB_CONN.cursor(cursor_factory=psycopg2.extras.DictCursor)
cursor.execute("SELECT id FROM feedjack_track")
results = cursor.fetchall()
for result in results:
track_ids.append(result['id'])
# Get "trackedphrase" for each "track"
for track_id in track_ids:
TRACK_DICT[track_id] = []
cursor.execute("SELECT id, name, mode " \
"FROM feedjack_trackedphrase " \
"WHERE track_id=%s", (track_id,))
results = cursor.fetchall()
for result in results:
TRACK_DICT[track_id].append({
'trackedphrase_id':
result['id'],
'name':
result['name'],
'mode':
result['mode']
})
LOGGER.info("Obtaining trackedphrase completed")
#pprint(TRACK_DICT)
except:
DB_CONN.rollback()
LOGGER.info(sys.exc_info())
def post_init(self):
#初始化 新建RabbitMQ 队列和exchange并绑定（2个）
LOGGER.info('Initializing a FeedProcessor')
#self.feedprocessor.post_init()
LOGGER.info('Initializing a FeedProcessor completed')
try:
LOGGER.info('Opening a connection')
#self.dispatcher = Dispatcher(self.options)
self.connection = SelectConnection(
parameters=settings.pika_parameters,
on_open_callback=self.on_connection_open)
try:
LOGGER.info('Starting ioloop')
self.connection.ioloop.start()
except KeyboardInterrupt:
# Gracefully close the connection
self.connection.close()
# Loop until we're fully closed, will stop on its own
self.connection.ioloop.start()
except:
(etype, eobj, etb) = sys.exc_info()
print traceback.format_exception(etype, eobj, etb)
def on_connection_open(self, unused_connection):
LOGGER.info('Opening a connection completed')
self.open_channel()
def open_channel(self):
LOGGER.info('Opening a channel')
self.connection.channel(on_open_callback=self.on_channel_open)
def on_channel_open(self, new_channel):
LOGGER.info('Opening a channel completed')
self.channel = new_channel
self.declare_exchange()
self.declare_exchange2()
def declare_exchange(self):
LOGGER.info('Declaring an exchange')
self.channel.exchange_declare(
exchange=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_NAME,
exchange_type=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_TYPE,
passive=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_PASSIVE,
durable=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_DURABLE,
auto_delete=settings.
RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_AUTO_DELETE,
internal=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_INTERNAL,
nowait=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_NOWAIT,
arguments=None, # Custom key/value pair arguments for the exchange
callback=self.on_exchange_declared
) # Call this method on Exchange.DeclareOk
def on_exchange_declared(self, unused_frame):
LOGGER.info('Declaring an exchange completed')
self.declare_queue()
def declare_queue(self):
LOGGER.info('Declaring a queue')
self.channel.queue_declare(
self.on_queue_declared,
settings.RABBITMQ_NEWSFEED_RAW_FEED_QUEUE_NAME)
def on_queue_declared(self, method_frame):
LOGGER.info('Declaring a queue completed')
self.bind_queue()
def bind_queue(self):
LOGGER.info('Binding a queue')
self.channel.queue_bind(
callback=self.on_queue_binded,
queue=settings.RABBITMQ_NEWSFEED_RAW_FEED_QUEUE_NAME,
exchange=settings.RABBITMQ_NEWSFEED_RAW_FEED_EXCHANGE_NAME,
routing_key=settings.RABBITMQ_NEWSFEED_RAW_FEED_ROUTING_KEY)
def on_queue_binded(self, frame):
LOGGER.info('Binding a queue completed')
# Start the thread's activity
self.start()
###############################
def declare_exchange2(self):
LOGGER.info('Declaring an exchange2')
self.channel.exchange_declare(
exchange=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NAME,
exchange_type=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_TYPE,
passive=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_PASSIVE,
durable=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_DURABLE,
auto_delete=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_AUTO_DELETE,
internal=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_INTERNAL,
nowait=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NOWAIT,
arguments=None, # Custom key/value pair arguments for the exchange
callback=self.on_exchange_declared2
) # Call this method on Exchange.DeclareOk
def on_exchange_declared2(self, frame):
LOGGER.info('Declaring an exchange2 completed')
self.declare_queue2()
def declare_queue2(self):
LOGGER.info('Declaring a queue2')
self.channel.queue_declare(
callback=self.on_queue_declared2,
queue=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NAME,
passive=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_PASSIVE,
durable=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_DURABLE,
exclusive=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_EXCLUSIVE,
auto_delete=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_AUTO_DELETE,
nowait=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NOWAIT,
arguments=None)
def on_queue_declared2(self, method_frame):
LOGGER.info('Declaring a queue2 completed')
self.bind_queue2()
def bind_queue2(self):
LOGGER.info('Binding a queue2')
self.channel.queue_bind(
callback=self.on_queue_binded2,
queue=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NAME,
exchange=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NAME,
routing_key=settings.RABBITMQ_NEWSFEED_ENTRY_ROUTING_KEY)
def on_queue_binded2(self, frame):
LOGGER.info('Binding a queue completed on 2')
def on_message(self, channel, basic_deliver, properties, body):
"""Invoked by pika when a message is delivered from RabbitMQ. The
channel is passed for your convenience. The basic_deliver object that
is passed in carries the exchange, routing key, delivery tag and
a redelivered flag for the message. The properties passed in is an
instance of BasicProperties with the message properties and the body
is the message that was sent
收到消息message，processfeed 类处理成单个postlist并发送给websocket
:param pika.channel.Channel channel: The channel object
:param pika.Spec.Basic.Deliver: basic_deliver method
:param pika.Spec.BasicProperties: properties
:param str|unicode body: The message body
"""
LOGGER.info('Received message # %s from %s',
basic_deliver.delivery_tag, properties.app_id)
self.ack_message(basic_deliver.delivery_tag)
try:
data = JSON_DECODER.decode(body)
#pprint(data)
(entries_status, post_list) = self.processfeed.process(data)
for post in post_list:
LOGGER.info('Publishing data (post id: %d) to %s',
post['post_id'],
settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NAME)
json_data = JSON_ENCODER.encode(post)
self.channel.basic_publish(
exchange=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NAME,
routing_key=settings.RABBITMQ_NEWSFEED_ENTRY_ROUTING_KEY,
body=json_data)
LOGGER.info('Publishing data completed')
except simplejson.JSONDecodeError:
LOGGER.info(sys.exc_info())
#except simplejson.JSONEncodeError:
# LOGGER.info(sys.exc_info())
def ack_message(self, delivery_tag):
"""Acknowledge the message delivery from RabbitMQ by sending a
Basic.Ack RPC method for the delivery tag.
:param int delivery_tag: The delivery tag from the Basic.Deliver frame
"""
LOGGER.info('Acknowledging message %s', delivery_tag)
self.channel.basic_ack(delivery_tag)
def run(self):
#设置channel.basic_consume MQ消息接收
LOGGER.info('Start to consume message from queue')
self.channel.basic_consume(
consumer_callback=self.on_message,
queue=settings.RABBITMQ_NEWSFEED_RAW_FEED_QUEUE_NAME,
no_ack=
False, # Set to True means tell the broker to not expect a response
exclusive=
False, # Set to True means don't allow other consumers on the queue
consumer_tag=None) # Specify your own consumer tag
class NewsFeedOpinion(Thread):
def __init__(self):
super(NewsFeedOpinion, self).__init__()
self.feeds = []
self.update_sentimentwords()
self.channel = None
self.connection = None
self.processopinion = ProcessOpinion()
self.alive = False
self.stop_flag = False
self.r = None
self.r_last_touched = datetime.now()
def update_sentimentwords(self):
global NEG_WORDS
global DB_CONN
try:
LOGGER.info("Obtaining sentiment word list from postgre")
if DB_CONN is None:
connect_pg()
cursor = DB_CONN.cursor()
cursor.execute(
"SELECT name FROM sentidict_sentimentword WHERE mode='2'")
results = cursor.fetchall()
for result in results:
NEG_WORDS.append(str(result[0]))
LOGGER.info("Obtaining sentiment word list from postgre completed")
except:
DB_CONN.rollback()
logger(sys.exc_info())
def post_init(self):
try:
LOGGER.info('Opening a connection')
self.connection = SelectConnection(
parameters=settings.pika_parameters,
on_open_callback=self.on_connection_open)
try:
LOGGER.info('Starting ioloop')
self.connection.ioloop.start()
except KeyboardInterrupt:
# Gracefully close the connection
self.connection.close()
# Loop until we're fully closed, will stop on its own
self.connection.ioloop.start()
except:
(etype, eobj, etb) = sys.exc_info()
print traceback.format_exception(etype, eobj, etb)
def on_connection_open(self, unused_connection):
LOGGER.info('Opening a connection completed')
self.open_channel()
def open_channel(self):
LOGGER.info('Opening a channel')
self.connection.channel(on_open_callback=self.on_channel_open)
def on_channel_open(self, new_channel):
LOGGER.info('Opening a channel completed')
self.channel = new_channel
self.declare_exchange()
def declare_exchange(self):
LOGGER.info('Declaring an exchange')
self.channel.exchange_declare(
exchange=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NAME,
exchange_type=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_TYPE,
passive=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_PASSIVE,
durable=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_DURABLE,
auto_delete=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_AUTO_DELETE,
internal=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_INTERNAL,
nowait=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NOWAIT,
arguments=None, # Custom key/value pair arguments for the exchange
callback=self.on_exchange_declared
) # Call this method on Exchange.DeclareOk
def on_exchange_declared(self, unused_frame):
LOGGER.info('Declaring an exchange completed')
self.declare_queue()
def declare_queue(self):
LOGGER.info('Declaring a queue')
self.channel.queue_declare(
callback=self.on_queue_declared,
queue=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NAME,
passive=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_PASSIVE,
durable=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_DURABLE,
exclusive=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_EXCLUSIVE,
auto_delete=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_AUTO_DELETE,
nowait=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NOWAIT,
arguments=None)
def on_queue_declared(self, method_frame):
LOGGER.info('Declaring a queue completed')
self.bind_queue()
def bind_queue(self):
LOGGER.info('Binding a queue')
self.channel.queue_bind(
callback=self.on_queue_binded,
queue=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NAME,
exchange=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NAME,
routing_key=settings.RABBITMQ_NEWSFEED_ENTRY_ROUTING_KEY)
def on_queue_binded(self, frame):
LOGGER.info('Binding a queue completed')
# Start the thread's activity
self.start()
def on_message(self, channel, basic_deliver, properties, body):
"""Invoked by pika when a message is delivered from RabbitMQ. The
channel is passed for your convenience. The basic_deliver object that
is passed in carries the exchange, routing key, delivery tag and
a redelivered flag for the message. The properties passed in is an
instance of BasicProperties with the message properties and the body
is the message that was sent.
:param pika.channel.Channel channel: The channel object
:param pika.Spec.Basic.Deliver: basic_deliver method
:param pika.Spec.BasicProperties: properties
:param str|unicode body: The message body
"""
LOGGER.info('Received message # %s from %s',
basic_deliver.delivery_tag, properties.app_id)
self.ack_message(basic_deliver.delivery_tag)
try:
data = JSON_DECODER.decode(body)
#pprint(data)
self.processopinion.process(data)
except simplejson.JSONDecodeError:
LOGGER.info(sys.exc_info())
def ack_message(self, delivery_tag):
"""Acknowledge the message delivery from RabbitMQ by sending a
Basic.Ack RPC method for the delivery tag.
:param int delivery_tag: The delivery tag from the Basic.Deliver frame
"""
LOGGER.info('Acknowledging message %s', delivery_tag)
self.channel.basic_ack(delivery_tag)
def run(self):
LOGGER.info('Start to consume message from queue')
self.channel.basic_consume(
consumer_callback=self.on_message,
queue=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NAME,
no_ack=
False, # Set to True means tell the broker to not expect a response
exclusive=
False, # Set to True means don't allow other consumers on the queue
consumer_tag=None) # Specify your own consumer tag
class dataChannel(object):
""" The dataChannel is the base class of all our datasource.
It's purpose is to: a). Setup the queues"""
def __init__(self,
server_name='test',
mq_exchange='',
mq_queue='',
mq_host=''):
self.channel = None
self.id = server_name
self.queue_counter = 0
self.queue = mq_queue
self.routing_key = ''
self.exchange = mq_exchange
self.connection = None
self.connected = False
self.connecting = False
self.rabbithost = mq_host
logging.getLogger('pika').setLevel(logging.DEBUG)
def get_connection(self):
return self.connection
def connect(self):
if self.connecting:
return
self.connecting = True
credentials = pika.PlainCredentials('guest', 'guest')
params = pika.ConnectionParameters(host="hackinista.com",
port=5672,
virtual_host="/",
credentials=credentials)
host = (len(sys.argv) > 1) and sys.argv[1] or '127.0.0.1'
try:
self.connection = SelectConnection(params, self.on_connected)
except:
# self.L.critical("Error connecting to rabbitmq on host =
# "+self.host);
sys.exit(-1)
###
def on_connected(self, connection):
self.connection = connection
self.connection.channel(self.on_channel_open)
self.connected = True
def on_channel_open(self, channel):
self.channel = channel
try:
self.channel.queue_declare(queue=self.queue,
auto_delete=False,
durable=True,
exclusive=False,
callback=self.on_queue_declared)
except:
print "Error declaring queue = " + self.queue
sys.exit(-1)
def on_queue_declared(self, frame):
try:
self.channel.queue_bind(exchange=self.exchange,
queue=self.queue,
routing_key=self.routing_key,
callback=self.on_queue_bound)
except:
print "Binding to queue = " + self.queue
pass
def on_queue_bound(self, frame):
self.channel.basic_consume(consumer_callback=self.handle_delivery,
queue=self.queue,
no_ack=False)
def handle_delivery(self, channel, method_frame, header_frame, body):
print "7...Basic.Deliver %s delivery-tag %i: %s" %\
(header_frame.content_type,
method_frame.delivery_tag,
body)
self.data_op(body)
channel.basic_ack(delivery_tag=method_frame.delivery_tag)
def data_op(self, args):
print "Please implement get_data"
class Amqp(object):
def __init__(self, conf):
# RabbitMQ general options
self.cacertfile = conf['cacertfile']
self.certfile = conf['certfile']
self.exchange = conf['exchange']
self.status_exchange = conf['status_exchange']
self.fail_if_no_peer_cert = conf['fail_if_no_peer_cert']
self.heartbeat = conf['heartbeat']
self.host = conf['host']
self.keyfile = conf['keyfile']
self.password = conf['password']
self.port = conf['port']
self.ssl_port = conf['ssl_port']
self.queue = conf['uuid']
self.retry_timeout = conf['retry_timeout']
self.ssl_auth = conf['ssl_auth']
self.use_ssl = conf['use_ssl']
self.username = conf['username']
self.vhost = conf['vhost']
# Connection and channel initialization
self.connection = None
self.channel = None
# Plain credentials
credentials = PlainCredentials(self.username, self.password)
pika_options = {
'host': self.host,
'port': self.port,
'virtual_host': self.vhost,
'credentials': credentials
}
# SSL options
if self.use_ssl:
pika_options['ssl'] = True
pika_options['port'] = self.ssl_port
if self.ssl_auth:
pika_options['credentials'] = ExternalCredentials()
pika_options['ssl_options'] = {
'ca_certs': self.cacertfile,
'certfile': self.certfile,
'keyfile': self.keyfile,
'cert_reqs': CERT_REQUIRED
}
if self.heartbeat:
pika_options['heartbeat'] = self.heartbeat
self.parameters = None
try:
self.parameters = pika.ConnectionParameters(**pika_options)
except TypeError as err:
self.logger.debug(err)
# Let's be compatible with original pika version (no integer for
# heartbeats and no ssl.
self.logger.warning("Wrong pika lib version, won't use ssl.")
pika_options['heartbeat'] = True
if self.use_ssl:
self.use_ssl = False
pika_options['port'] = self.port
del pika_options['ssl']
if self.ssl_auth:
self.ssl_auth = False
del pika_options['ssl_options']
self.parameters = pika.ConnectionParameters(**pika_options)
def connect(self):
SelectPoller.TIMEOUT = .1
self.connection = SelectConnection(self.parameters, self.on_connected)
self.connection.ioloop.start()
def close(self, amqperror=False):
if (self.connection and not self.connection.closing
and not self.connection.closed):
self.logger.debug("Closing connection")
self.connection.close()
#self.connection.ioloop.start()
def on_remote_close(self, code, text):
self.logger.debug("Remote channel close, code %d" % code)
time.sleep(2)
if code != 200:
self.close()
raise AmqpError(text)
def on_connection_closed(self, frame):
self.connection.ioloop.stop()
def on_connected(self, connection):
self.connection = connection
self.connection.add_on_close_callback(self.on_connection_closed)
self.connection.channel(self.on_channel_open)
