def enqueue(self, urls):
if not urls:
return
for url, score in urls:
Page.add_page(self.db, self.cache, url, score, self.async_get_func,
self.publish, self.config, self.girl,
self.violation_definitions, self.handle_page_added)
self.wait_for_async_requests()
def post(self):
post_data = loads(self.request.body)
url = post_data["url"]
score = float(post_data.get("score", self.application.config.DEFAULT_PAGE_SCORE))
result = yield Page.add_page(
self.db,
self.application.cache,
url,
score,
self.application.http_client.fetch,
self.application.event_bus.publish,
self.application.config,
)
created, url, result = result
if not created and result["reason"] == "invalid_url":
self.set_status(400, "Invalid url [%s]" % url)
self.write_json(
{"reason": "invalid_url", "url": url, "status": result["status"], "details": result["details"]}
)
return
if not created and result["reason"] == "redirect":
self.set_status(400, "Redirect URL [%s]" % url)
self.write_json({"reason": "redirect", "url": url, "effectiveUrl": result["effectiveUrl"]})
return
self.write(str(result))
self.finish()
def enqueue(self, urls):
if not urls:
return
for url, score in urls:
Page.add_page(
self.db,
self.cache,
url,
score,
self.async_get_func,
self.publish,
self.config,
self.handle_page_added
)
self.wait_for_async_requests()
def post(self):
post_data = loads(self.request.body)
url = post_data['url']
score = float(
post_data.get('score', self.application.config.DEFAULT_PAGE_SCORE))
result = yield Page.add_page(
self.db, self.application.cache, url, score,
self.application.http_client.fetch,
self.application.event_bus.publish, self.application.config,
self.application.girl, self.application.default_violations_values,
self.application.violation_definitions)
created, url, result = result
if not created and result['reason'] == 'invalid_url':
self.set_status(400, self._('Invalid url [%s]') % url)
self.write_json({
'reason': 'invalid_url',
'url': url,
'status': result['status'],
'details': result['details']
})
return
if not created and result['reason'] == 'redirect':
self.set_status(400,
self._('Supplied URL is a redirect [%s]') % url)
self.write_json({
'reason': 'redirect',
'url': url,
'effectiveUrl': result['effectiveUrl']
})
return
yield self.application.cache.add_next_job_bucket(result, url)
self.write(str(result))
self.finish()
def post(self):
post_data = loads(self.request.body)
url = post_data['url']
score = float(post_data.get('score', self.application.config.DEFAULT_PAGE_SCORE))
result = yield Page.add_page(
self.db,
self.application.cache,
url,
score,
self.application.http_client.fetch,
self.application.event_bus.publish,
self.application.config
)
created, url, result = result
if not created and result['reason'] == 'invalid_url':
self.set_status(400, 'Invalid url [%s]' % url)
self.write_json({
'reason': 'invalid_url',
'url': url,
'status': result['status'],
'details': result['details']
})
return
if not created and result['reason'] == 'redirect':
self.set_status(400, 'Redirect URL [%s]' % url)
self.write_json({
'reason': 'redirect',
'url': url,
'effectiveUrl': result['effectiveUrl']
})
return
self.write(str(result))
self.finish()
def post(self):
post_data = loads(self.request.body)
url = post_data["url"]
score = float(post_data.get("score", self.application.config.DEFAULT_PAGE_SCORE))
result = yield Page.add_page(
self.db,
self.application.cache,
url,
score,
self.application.http_client.fetch,
self.application.event_bus.publish,
self.application.config,
self.application.girl,
self.application.default_violations_values,
self.application.violation_definitions,
)
created, url, result = result
if not created and result["reason"] == "invalid_url":
self.set_status(400, self._("Invalid url [%s]") % url)
self.write_json(
{"reason": "invalid_url", "url": url, "status": result["status"], "details": result["details"]}
)
return
if not created and result["reason"] == "redirect":
self.set_status(400, self._("Supplied URL is a redirect [%s]") % url)
self.write_json({"reason": "redirect", "url": url, "effectiveUrl": result["effectiveUrl"]})
return
yield self.application.cache.add_next_job_bucket(result, url)
self.write(str(result))
self.finish()