class Lead(object):
"Redis publisher."
def __init__(self, channel, queue=False,
host='localhost', port=6379, db=0, password=None):
self.channel = channel
self.queue = queue
self.redis = Redis(host=host, port=port, db=db, password=password)
def _block(self, uuid):
self.redis.subscribe(uuid)
for message in self.redis.listen():
if message['type'] == 'message':
action = message['data']
if action == "COMPLETE":
self.redis.unsubscribe(uuid)
else:
print action
retval = self.redis.get("understudy:result:%s" % uuid)
self.redis.delete("understudy:result:%s" % uuid)
self.redis.delete("understudy:log:%s" % uuid)
return retval
def _handle(self, directive, block):
serialized = simplejson.dumps(directive)
if self.queue:
self.redis.rpush(self.channel, serialized)
self.redis.publish(self.channel, "GO!")
else:
understudies = self.redis.publish(self.channel, serialized)
if not understudies:
raise NoUnderstudiesError
if block:
return self._block(directive['uuid'])
else:
return Result(directive['uuid'], self.redis)
def shell(self, command, block=False):
uuid = str(uuid4())
directive = {'uuid':uuid, 'handler':'shell', 'action':command}
return self._handle(directive, block)
def perform(self, action, block=False):
uuid = str(uuid4())
directive = {'uuid':uuid, 'handler':'function', 'action':action}
return self._handle(directive, block)
from uuid import uuid4
siteList = ["http://www.cnn.com/2010/POLITICS/08/13/democrats.social.security/index.html",
"http://www.huffingtonpost.com/2010/08/13/federal-reserve-pursuing_n_681540.html",
"http://techcrunch.com/2010/08/13/gantto-takes-on-microsoft-project-with-web-based-project-management-application/",
]
redis = Redis("localhost")
siteList.extend(siteList)
siteList.extend(siteList)
siteList.extend(siteList)
for site in siteList:
sid = str(uuid4())
redis.push('readability:jobs', dumps({
'id':sid,
'url':site}))
redis.subscribe(sid)
gen = redis.listen() ; next(gen)
result = None
try:
result = next(gen)['data']
print "ok"
finally:
redis.unsubscribe() ; next(gen)
class Understudy(object):
"Redis subscriber/queue processor."
HANDLERS = {'shell':ShellHandler,
'function':FunctionHandler,}
def __init__(self, channel, queue=False,
host='localhost', port=6379, db=0, password=None):
self.channel = channel
self.queue = queue
self.redis = Redis(host=host,
port=port,
db=db,
password=password)
self.subscriber = Redis(host=host,
port=port,
db=db,
password=password)
self.subscriber.subscribe(self.channel)
def process_queue(self):
msg = self.redis.lpop(self.channel)
while msg:
message = simplejson.loads(msg)
self.process_message(message)
msg = self.redis.lpop(self.channel)
def process_message(self, message):
uuid = message['uuid']
handler = message['handler']
action = message['action']
logger = logging.getLogger(uuid)
logger.setLevel(logging.INFO)
log_handler = UnderstudyHandler(uuid, self.redis)
logger.addHandler(log_handler)
cls = Understudy.HANDLERS[handler]
handler = cls(uuid, action, logger=logger)
retval = handler.perform()
self.redis.set("understudy:result:%s" % uuid, retval)
self.redis.publish(uuid, "COMPLETE")
def start(self):
if self.queue:
self.process_queue()
for message in self.subscriber.listen():
if message['type'] == 'subscribe':
continue
if self.queue:
self.process_queue()
continue
message = simplejson.loads(message['data'])
self.process_message(message)
def stop(self):
self.subscriber.unsubscribe(self.channel)
