def send_empty_result(self, crawl_request, e):
result = addict.Dict()
result.body = str(e)
result.status_code = 500
result.url = crawl_request.url
result.crawl_task = crawl_request
result.actions = crawl_request.actions
result.crawal_time = datetime.now(timezone.utc).isoformat()
toddler.send_message_sync(
self._rabbitmq_url,
ujson.dumps(result.to_dict()),
exchange=self._exchange,
routing_key=self._routing_key
)
def send_message(self, msg, exchange):
return send_message_sync(
self._rabbitmq_url,
ujson.dumps(msg),
exchange,
exchange
)
def send_result(self, req, crawl_request):
result = addict.Dict()
result.body = req.text
result.cookies = [(key, val)
for key, val in req.cookies.get_dict().items()]
result.url = crawl_request.url
result.crawl_task = crawl_request
result.actions = crawl_request.actions
result.headers = req.headers
result.status_code = req.status_code
result.crawl_time = datetime.now(timezone.utc).isoformat()
toddler.send_message_sync(
self._rabbitmq_url, # this is the rabbit url
ujson.dumps(result.to_dict()),
exchange=self._exchange,
routing_key=self._routing_key
)
