def publish(self, message, priority=0):
if priority < 0:
raise MQError("priority %d should be >= 0" % priority)
if priority >= self._priority_level:
logging.warn("priority has been changed due to exceeding", \
message=message, original_priority=priority, \
changed_priority=self._priority_level - 1)
priority = self._priority_level - 1
if priority < 0:
priority = 0
#determine properties
properties = pika.BasicProperties()
properties.content_type = self._content_type
if self._persistent:
properties.delivery_mode = 2
if self._with_timestamp:
properties.timestamp = datetime2timestamp(
datetime.datetime.utcnow())
#get group_id
if self._group_mode:
if self._content_type != "text/json":
#Note: here we just support hashable object, e.g., str/unicode etc.
group_hash = message
else:
if not ("__group_hash" in message.keys()):
raise MQError(
"__group_hash is required if message type is in group_mode"
)
group_hash = message["__group_hash"]
group_int_hash = hash(group_hash)
group_id = group_int_hash % self._group_counts[priority]
else:
group_id = 0
#the message is changed here
if self._content_type == "text/json" and "__group_hash" in message.keys(
):
message.pop("__group_hash")
#determine body
if self._content_type == "text/json":
body = dumps_jsonx(message)
else:
body = message
#determine routing_key
routing_key = self.generate_routing_name(self._binding_prefix,
priority, group_id)
try:
self._channel.basic_publish(exchange=self._exchange,
routing_key=routing_key,
body=body,
properties=properties)
except AttributeError, e:
logging.info('basic publish raise AttributeError: %s' % e)
raise pika.exceptions.AMQPConnectionError
def expire_message(self, message):
if not self._timestamp_expires:
raise MQError("expire_message should be valid when _timestamp_expires is true")
if self._aux_store is None:
logging.warn("aux_store is none in expire_message")
return
message_id = generate_message_id(message, self._message_ids)
timestamp = datetime2timestamp(datetime.datetime.utcnow())
self._aux_store.add_expired_message(self._message_type, message_id, timestamp)
def expire_message(self, message):
if not self._timestamp_expires:
raise MQError(
"expire_message should be valid when _timestamp_expires is true"
)
if self._aux_store is None:
logging.warn("aux_store is none in expire_message")
return
message_id = generate_message_id(message, self._message_ids)
timestamp = datetime2timestamp(datetime.datetime.utcnow())
self._aux_store.add_expired_message(self._message_type, message_id,
timestamp)
def publish(self, message, priority=0):
if priority < 0:
raise MQError("priority %d should be >= 0" % priority)
if priority >= self._priority_level:
logging.warn("priority has been changed due to exceeding", \
message=message, original_priority=priority, \
changed_priority=self._priority_level - 1)
priority = self._priority_level - 1
if priority < 0:
priority = 0
#determine properties
properties = pika.BasicProperties()
properties.content_type = self._content_type
if self._persistent:
properties.delivery_mode = 2
if self._with_timestamp:
properties.timestamp = datetime2timestamp(datetime.datetime.utcnow())
#get group_id
if self._group_mode:
if self._content_type != "text/json":
#Note: here we just support hashable object, e.g., str/unicode etc.
group_hash = message
else:
if not ("__group_hash" in message.keys()):
raise MQError("__group_hash is required if message type is in group_mode")
group_hash = message["__group_hash"]
group_int_hash = hash(group_hash)
group_id = group_int_hash % self._group_counts[priority]
else:
group_id = 0
#the message is changed here
if self._content_type == "text/json" and "__group_hash" in message.keys():
message.pop("__group_hash")
#determine body
if self._content_type == "text/json":
body = dumps_jsonx(message)
else:
body = message
#determine routing_key
routing_key = self.generate_routing_name(self._binding_prefix, priority, group_id)
try:
self._channel.basic_publish(exchange=self._exchange, routing_key=routing_key, body=body, properties=properties)
except AttributeError, e:
logging.info('basic publish raise AttributeError: %s' % e)
raise pika.exceptions.AMQPConnectionError
def get_content(channel):
category = CHANNEL_MAPS.get(channel)
r = requests.get(API.format(category, datetime2timestamp()))
data = r.json()['data']
for i in data:
text = (u'<{seo_url}|{title}> 赞{bury_count} 踩{digg_count} - '
'{source} {datetime}').format(**i)
image_url = i.get('middle_image', '')
if isinstance(image_url, dict):
image_url = image_url['url']
attach = gen_attachment(trunc_utf8(i['abstract']), image_url,
image_type='thumb', title=i['title'],
title_link=i['seo_url'], fallback=False)
yield text, attach
def _on_channel_open(self, channel):
'''
the main work is done here
'''
logging.info('begin publish the message ')
while not self._stop_event.is_set():
if self._unfinished_message == None:
message = self._get_one_message() #get a message from the queue
self._unfinished_message = message
else:
message = self._unfinished_message
# prision break the block queue
if message != None:
message_key = message.get('message_key', None)
message_body = message.get('message_body', None)
if message_key == None:
logging.error('empty message_key')
continue
if message_body == None:
logging.warn('empty message_body')
message_body = ''
# make the exchange queue routing_key as the message_config
exchange = self._declare_exchange(message_key)
queue = self._declare_queue(message_key, message_body)
routing_key = self._bind_exchange(message_key, message_body)
# make the message parameters
message_parameters = self._message_config.get(message_key, self._message_config['__default_message_config'])
properties = pika.BasicProperties()
if message_parameters.get('content_type', 'text/json') == 'text/json':
properties.content_type = 'text/json'
message = dumps_jsonx(message_body)
if message_parameters.get('persistent', True):
properties.delivery_mode = 2
if message_parameters.get('with_timestamp', False):
properties.timestamp = datetime2timestamp(datetime.datetime.utcnow())
# publish the message use select_connecter
self._publish_message(exchange, routing_key, message, properties)
# call handler when publish the message
if self._process_handler != None:
self._process_handler(message_key, message_body)
# clear the unfinished message
self._unfinished_message = None
else:
break
def get_content(channel):
category = CHANNEL_MAPS.get(channel)
r = requests.get(API.format(category, datetime2timestamp()))
data = r.json()['data']
for i in data:
text = (u'<{seo_url}|{title}> 赞{bury_count} 踩{digg_count} - '
'{source} {datetime}').format(**i)
image_url = i.get('middle_image', '')
if isinstance(image_url, dict):
image_url = image_url['url']
attach = gen_attachment(trunc_utf8(i['abstract']),
image_url,
image_type='thumb',
title=i['title'],
title_link=i['seo_url'],
fallback=False)
yield text, attach
