class Producer():
def __init__(self, message_key, message_auth=None):
self.handlerRepository = HandlerRepository(False)
if message_auth != None:
self.handlerRepository.init_Message(message_key, message_auth)
self.message_key = message_key
def start(self, log_config):
# while True:
self.process()
def process(self):
now = int(time.time())
msg = {
'url': 'http://img1.gtimg.com/news/pics/hv1/59/206/1506/97980239.jpg',
'method': 'GET',
'priority': 1,
'headers': {},
'params': {},
'__priority': 1,
'__delivery_tag': 1
}
time.sleep(1)
if self.handlerRepository.process(self.message_key, msg):
print '[Publish %s] received message: %s %s' % (threading.currentThread(), self.message_key, msg)
class Producer():
def __init__(self, message_key, message_auth=None):
self.handlerRepository = HandlerRepository(False)
if message_auth != None:
self.handlerRepository.init_Message(message_key, message_auth)
self.message_key = message_key
def start(self, log_config):
# while True:
self.process()
def process(self):
now = int(time.time())
msg = {
'url':
'http://img1.gtimg.com/news/pics/hv1/59/206/1506/97980239.jpg',
'method': 'GET',
'priority': 1,
'headers': {},
'params': {},
'__priority': 1,
'__delivery_tag': 1
}
time.sleep(1)
if self.handlerRepository.process(self.message_key, msg):
print '[Publish %s] received message: %s %s' % (
threading.currentThread(), self.message_key, msg)
class Producer:
def __init__(self, message_key, message_auth=None):
self.handlerRepository = HandlerRepository()
if message_auth != None:
self.handlerRepository.init_Message(message_key, message_auth)
self.message_key = message_key
def start(self, log_config):
while True:
self.process()
def process(self):
now = int(time.time())
msg = {"id": now, "name": "producer %s" % (now % 10), "phone": now, "__priority": (now % 3)}
if self.handlerRepository.process(self.message_key, msg):
print "[Publish %d] received message: %s %s" % (os.getpid(), self.message_key, msg)
class Producer():
def __init__(self, message_key, message_auth=None):
self.handlerRepository = HandlerRepository()
if message_auth != None:
self.handlerRepository.init_Message(message_key, message_auth)
self.message_key = message_key
def start(self, log_config):
while True:
self.process()
def process(self):
now = int(time.time())
msg = {
'id': now,
'name': 'producer %s' % (now % 10),
'phone': now,
'__priority': (now % 3),
}
if self.handlerRepository.process(self.message_key, msg):
print '[Publish %d] received message: %s %s' % (
os.getpid(), self.message_key, msg)
'''
Created on: Dec 20, 2013
@author: qwang
'''
import re
import datetime
import logging
from weibonews.settings import TYPE_MAP
from weibonews.utils.format import timestamp2datetime
from weibonews.utils.decorators import perf_logging
from messagequeue.HandlerRepository import HandlerRepository
_LOGGER = logging.getLogger('weibonews.external')
_handler_repository = HandlerRepository()
_CONTENT_LENGTH_THRESHOLD = 100
_INCOMPLETE_CONTENT_LENGTH_THRESHOLD = 1000
_PUBDATE_INTERVAL = 1 # do not publish if published 1 days ago
_CLEAN_IMG_RE = re.compile(
"<dolphinimagestart--([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}--dolphinimageend>",
re.I)
@perf_logging
def _should_publish(info):
'''
Should publish the given info, return two value: should_publish and is_content_incomplete
'''
info_id = info['_id']
def __init__(self, message_key, message_auth=None):
self.handlerRepository = HandlerRepository()
if message_auth != None:
self.handlerRepository.init_Message(message_key, message_auth)
self.message_key = message_key
def __init__(self, message_key, message_auth=None):
self.handlerRepository = HandlerRepository(False)
if message_auth != None:
self.handlerRepository.init_Message(message_key, message_auth)
self.message_key = message_key