Exemplo n.º 1
0
class Worker(object):
      def __init__(self, queue_name = 'crawling'):
	  self.amqp = Amqp(queue_name, ['database'])
          self.crawler = Crawler(self.amqp)
	  self.amqp.receive(self)
       
      def __call__(self, ch, method, properties, body):
	  body = json.loads(body)
	  print 'Search for tag %s to maximum depth : %s'\
	      % (body['tag'], body['max_depth'])
	  self.crawler(body['job_id'], body['urls'], body['tag'], body['max_depth'])
	  ch.basic_ack(delivery_tag = method.delivery_tag)
Exemplo n.º 2
0
class Worker(object):
    def __init__(self, queue_name='database'):
        self.dbmgr = DbMgr()
        self.amqp = Amqp(queue_name, ['front_end'])
        self.amqp.receive(self)

    def __call__(self, ch, method, properties, body):
        body = json.loads(body)
        result = getattr(self.dbmgr, body['request'])(body)
        if result != None:
            result = json.dumps(result)
            self.amqp.send('web_frontend', result)
        ch.basic_ack(delivery_tag=method.delivery_tag)
Exemplo n.º 3
0
class Worker(object):
    def __init__(self, queue_name='crawling'):
        self.amqp = Amqp(queue_name, ['database'])
        self.crawler = Crawler(self.amqp)
        self.amqp.receive(self)

    def __call__(self, ch, method, properties, body):
        body = json.loads(body)
        print 'Search for tag %s to maximum depth : %s'\
            % (body['tag'], body['max_depth'])
        self.crawler(body['job_id'], body['urls'], body['tag'],
                     body['max_depth'])
        ch.basic_ack(delivery_tag=method.delivery_tag)
Exemplo n.º 4
0
class Worker(object):
      def __init__(self, queue_name = 'database'):
          self.dbmgr = DbMgr()
	  self.amqp = Amqp(queue_name, ['front_end'])
	  self.amqp.receive(self)
 
      def __call__(self, ch, method, properties, body):
	  body = json.loads(body)
	  result = getattr(self.dbmgr, body['request'])(body)
	  if result != None:
	    result = json.dumps(result)
	    self.amqp.send('web_frontend', result)
	  ch.basic_ack(delivery_tag = method.delivery_tag)
Exemplo n.º 5
0
class GetJobResult(tornado.web.RequestHandler):
      def __init__(self, *args, **kw):
	  super(GetJobResult, self).__init__(*args, **kw)
	  self.amqp = Amqp('web_frontend', ['database'])

      def get(self, job_id):
	  print "Get result of [%s]" % job_id
	  self.amqp.send('database', json.dumps({'request' : 'result',
						 'job_id' : job_id}))
	  self.amqp.receive(self.__get_results)
	  self.set_header("Content-Type", "application/json")
	  self.write(self.__answer)

      def __get_results(self, ch, method, properties, body):
	  self.__answer = body
	  ch.basic_ack(delivery_tag = method.delivery_tag)
	  ch.stop_consuming()
Exemplo n.º 6
0
class GetJobResult(tornado.web.RequestHandler):
    def __init__(self, *args, **kw):
        super(GetJobResult, self).__init__(*args, **kw)
        self.amqp = Amqp('web_frontend', ['database'])

    def get(self, job_id):
        print "Get result of [%s]" % job_id
        self.amqp.send('database',
                       json.dumps({
                           'request': 'result',
                           'job_id': job_id
                       }))
        self.amqp.receive(self.__get_results)
        self.set_header("Content-Type", "application/json")
        self.write(self.__answer)

    def __get_results(self, ch, method, properties, body):
        self.__answer = body
        ch.basic_ack(delivery_tag=method.delivery_tag)
        ch.stop_consuming()
Exemplo n.º 7
0
class PushJob(tornado.web.RequestHandler):
    def __init__(self, *args, **kw):
        super(PushJob, self).__init__(*args, **kw)
        self.amqp = Amqp('web_frontend', ['database', 'crawling'])

    def get(self):
        self.write("usage :<br/>Use curl to send some post request.")

    def post(self):
        request = json.loads(self.request.body)
        self.amqp.send('database', json.dumps({'request': 'new_job'}))
        self.amqp.receive(self.__get_new_job)
        self.amqp.send(
            'crawling',
            json.dumps({
                'job_id': self.__new_job_id,
                'urls': request['urls'],
                'tag': 'img',
                'max_depth': 1
            }))
        self.set_header("Content-Type", "application/json")
        self.write(json.dumps({'job_id': self.__new_job_id}))

    def __get_new_job(self, ch, method, properties, body):
        result = json.loads(body)
        self.__new_job_id = result['job_id']
        ch.basic_ack(delivery_tag=method.delivery_tag)
        ch.stop_consuming()
Exemplo n.º 8
0
class PushJob(tornado.web.RequestHandler):
      def __init__(self, *args, **kw):
	  super(PushJob, self).__init__(*args, **kw)
	  self.amqp = Amqp('web_frontend', ['database', 'crawling'])

      def get(self):
	  self.write("usage :<br/>Use curl to send some post request.")

      def post(self):
	  request = json.loads(self.request.body)
	  self.amqp.send('database', json.dumps({'request' : 'new_job'}))
	  self.amqp.receive(self.__get_new_job)
	  self.amqp.send('crawling', json.dumps({'job_id' : self.__new_job_id,
						 'urls' : request['urls'],
						 'tag' : 'img',
						 'max_depth' : 1}))
	  self.set_header("Content-Type", "application/json")
	  self.write(json.dumps({'job_id' : self.__new_job_id}))
      
      def __get_new_job(self, ch, method, properties, body):
	  result = json.loads(body)
	  self.__new_job_id = result['job_id']
	  ch.basic_ack(delivery_tag = method.delivery_tag)
	  ch.stop_consuming()
Exemplo n.º 9
0
 def __init__(self, *args, **kw):
     super(GetJobResult, self).__init__(*args, **kw)
     self.amqp = Amqp('web_frontend', ['database'])
Exemplo n.º 10
0
 def __init__(self, *args, **kw):
     super(PushJob, self).__init__(*args, **kw)
     self.amqp = Amqp('web_frontend', ['database', 'crawling'])
Exemplo n.º 11
0
 def __init__(self, queue_name='database'):
     self.dbmgr = DbMgr()
     self.amqp = Amqp(queue_name, ['front_end'])
     self.amqp.receive(self)
Exemplo n.º 12
0
      def __init__(self, *args, **kw):
	  super(GetJobResult, self).__init__(*args, **kw)
	  self.amqp = Amqp('web_frontend', ['database'])
Exemplo n.º 13
0
      def __init__(self, *args, **kw):
	  super(PushJob, self).__init__(*args, **kw)
	  self.amqp = Amqp('web_frontend', ['database', 'crawling'])
Exemplo n.º 14
0
 def __init__(self, queue_name='crawling'):
     self.amqp = Amqp(queue_name, ['database'])
     self.crawler = Crawler(self.amqp)
     self.amqp.receive(self)
Exemplo n.º 15
0
      def __init__(self, queue_name = 'database'):
          self.dbmgr = DbMgr()
	  self.amqp = Amqp(queue_name, ['front_end'])
	  self.amqp.receive(self)
Exemplo n.º 16
0
      def __init__(self, queue_name = 'crawling'):
	  self.amqp = Amqp(queue_name, ['database'])
          self.crawler = Crawler(self.amqp)
	  self.amqp.receive(self)