class Worker(object):
def __init__(self, queue_name = 'crawling'):
self.amqp = Amqp(queue_name, ['database'])
self.crawler = Crawler(self.amqp)
self.amqp.receive(self)
def __call__(self, ch, method, properties, body):
body = json.loads(body)
print 'Search for tag %s to maximum depth : %s'\
% (body['tag'], body['max_depth'])
self.crawler(body['job_id'], body['urls'], body['tag'], body['max_depth'])
ch.basic_ack(delivery_tag = method.delivery_tag)
class Worker(object):
def __init__(self, queue_name='database'):
self.dbmgr = DbMgr()
self.amqp = Amqp(queue_name, ['front_end'])
self.amqp.receive(self)
def __call__(self, ch, method, properties, body):
body = json.loads(body)
result = getattr(self.dbmgr, body['request'])(body)
if result != None:
result = json.dumps(result)
self.amqp.send('web_frontend', result)
ch.basic_ack(delivery_tag=method.delivery_tag)
class Worker(object):
def __init__(self, queue_name='crawling'):
self.amqp = Amqp(queue_name, ['database'])
self.crawler = Crawler(self.amqp)
self.amqp.receive(self)
def __call__(self, ch, method, properties, body):
body = json.loads(body)
print 'Search for tag %s to maximum depth : %s'\
% (body['tag'], body['max_depth'])
self.crawler(body['job_id'], body['urls'], body['tag'],
body['max_depth'])
ch.basic_ack(delivery_tag=method.delivery_tag)
class Worker(object):
def __init__(self, queue_name = 'database'):
self.dbmgr = DbMgr()
self.amqp = Amqp(queue_name, ['front_end'])
self.amqp.receive(self)
def __call__(self, ch, method, properties, body):
body = json.loads(body)
result = getattr(self.dbmgr, body['request'])(body)
if result != None:
result = json.dumps(result)
self.amqp.send('web_frontend', result)
ch.basic_ack(delivery_tag = method.delivery_tag)
class GetJobResult(tornado.web.RequestHandler):
def __init__(self, *args, **kw):
super(GetJobResult, self).__init__(*args, **kw)
self.amqp = Amqp('web_frontend', ['database'])
def get(self, job_id):
print "Get result of [%s]" % job_id
self.amqp.send('database', json.dumps({'request' : 'result',
'job_id' : job_id}))
self.amqp.receive(self.__get_results)
self.set_header("Content-Type", "application/json")
self.write(self.__answer)
def __get_results(self, ch, method, properties, body):
self.__answer = body
ch.basic_ack(delivery_tag = method.delivery_tag)
ch.stop_consuming()
class GetJobResult(tornado.web.RequestHandler):
def __init__(self, *args, **kw):
super(GetJobResult, self).__init__(*args, **kw)
self.amqp = Amqp('web_frontend', ['database'])
def get(self, job_id):
print "Get result of [%s]" % job_id
self.amqp.send('database',
json.dumps({
'request': 'result',
'job_id': job_id
}))
self.amqp.receive(self.__get_results)
self.set_header("Content-Type", "application/json")
self.write(self.__answer)
def __get_results(self, ch, method, properties, body):
self.__answer = body
ch.basic_ack(delivery_tag=method.delivery_tag)
ch.stop_consuming()
class PushJob(tornado.web.RequestHandler):
def __init__(self, *args, **kw):
super(PushJob, self).__init__(*args, **kw)
self.amqp = Amqp('web_frontend', ['database', 'crawling'])
def get(self):
self.write("usage :<br/>Use curl to send some post request.")
def post(self):
request = json.loads(self.request.body)
self.amqp.send('database', json.dumps({'request': 'new_job'}))
self.amqp.receive(self.__get_new_job)
self.amqp.send(
'crawling',
json.dumps({
'job_id': self.__new_job_id,
'urls': request['urls'],
'tag': 'img',
'max_depth': 1
}))
self.set_header("Content-Type", "application/json")
self.write(json.dumps({'job_id': self.__new_job_id}))
def __get_new_job(self, ch, method, properties, body):
result = json.loads(body)
self.__new_job_id = result['job_id']
ch.basic_ack(delivery_tag=method.delivery_tag)
ch.stop_consuming()
class PushJob(tornado.web.RequestHandler):
def __init__(self, *args, **kw):
super(PushJob, self).__init__(*args, **kw)
self.amqp = Amqp('web_frontend', ['database', 'crawling'])
def get(self):
self.write("usage :<br/>Use curl to send some post request.")
def post(self):
request = json.loads(self.request.body)
self.amqp.send('database', json.dumps({'request' : 'new_job'}))
self.amqp.receive(self.__get_new_job)
self.amqp.send('crawling', json.dumps({'job_id' : self.__new_job_id,
'urls' : request['urls'],
'tag' : 'img',
'max_depth' : 1}))
self.set_header("Content-Type", "application/json")
self.write(json.dumps({'job_id' : self.__new_job_id}))
def __get_new_job(self, ch, method, properties, body):
result = json.loads(body)
self.__new_job_id = result['job_id']
ch.basic_ack(delivery_tag = method.delivery_tag)
ch.stop_consuming()
def __init__(self, *args, **kw):
super(GetJobResult, self).__init__(*args, **kw)
self.amqp = Amqp('web_frontend', ['database'])
def __init__(self, *args, **kw):
super(PushJob, self).__init__(*args, **kw)
self.amqp = Amqp('web_frontend', ['database', 'crawling'])
def __init__(self, queue_name='database'):
self.dbmgr = DbMgr()
self.amqp = Amqp(queue_name, ['front_end'])
self.amqp.receive(self)
def __init__(self, *args, **kw):
super(GetJobResult, self).__init__(*args, **kw)
self.amqp = Amqp('web_frontend', ['database'])
def __init__(self, *args, **kw):
super(PushJob, self).__init__(*args, **kw)
self.amqp = Amqp('web_frontend', ['database', 'crawling'])
def __init__(self, queue_name='crawling'):
self.amqp = Amqp(queue_name, ['database'])
self.crawler = Crawler(self.amqp)
self.amqp.receive(self)
def __init__(self, queue_name = 'database'):
self.dbmgr = DbMgr()
self.amqp = Amqp(queue_name, ['front_end'])
self.amqp.receive(self)
def __init__(self, queue_name = 'crawling'):
self.amqp = Amqp(queue_name, ['database'])
self.crawler = Crawler(self.amqp)
self.amqp.receive(self)