예제 #1
0
파일: crawler.py 프로젝트: ifrpl/toddler
 def send_empty_result(self, crawl_request, e):
     result = addict.Dict()
     result.body = str(e)
     result.status_code = 500
     result.url = crawl_request.url
     result.crawl_task = crawl_request
     result.actions = crawl_request.actions
     result.crawal_time = datetime.now(timezone.utc).isoformat()
     toddler.send_message_sync(
         self._rabbitmq_url,
         ujson.dumps(result.to_dict()),
         exchange=self._exchange,
         routing_key=self._routing_key
     )
예제 #2
0
    def send_message(self, msg, exchange):

        return send_message_sync(
            self._rabbitmq_url,
            ujson.dumps(msg),
            exchange,
            exchange
        )
예제 #3
0
파일: crawler.py 프로젝트: ifrpl/toddler
    def send_result(self, req, crawl_request):

        result = addict.Dict()

        result.body = req.text
        result.cookies = [(key, val)
                          for key, val in req.cookies.get_dict().items()]
        result.url = crawl_request.url
        result.crawl_task = crawl_request
        result.actions = crawl_request.actions
        result.headers = req.headers
        result.status_code = req.status_code
        result.crawl_time = datetime.now(timezone.utc).isoformat()
        toddler.send_message_sync(
            self._rabbitmq_url,  # this is the rabbit url
            ujson.dumps(result.to_dict()),
            exchange=self._exchange,
            routing_key=self._routing_key
        )