def post(self):
if self.is_json:
if not checkToken(self.key, self.json_args):
return self.redirect("/login")
job_id = self.json_args['job_id']
self.control_job(job_id)
self.write({"status": "ok"})
else:
data = {}
data['token'] = self.get_body_argument('token')
if not checkToken(self.key, data):
return self.redirect("/login")
job_id = self.get_body_argument('job_id')
self.control_job(job_id)
self.redirect("/")
def post(self):
if self.is_json:
if not checkToken(self.key, self.json_args):
return self.redirect("/login")
spider_id = self.json_args.get('_id', None)
if self.remove_spiders(spider_id):
self.domain_crawlers.get_spider(self.spider_storage)
self.write({"status": "ok"})
else:
self.write({"status": "error"})
def post(self):
if self.is_json:
if not checkToken(self.key, self.json_args):
return self.redirect("/login")
spider_id = self.json_args.get('_id', None)
domain = self.json_args['domain']
args = self.json_args.get('options', {}).get('args', {})
settings = self.json_args.get('options', {}).get('settings', {})
if self.save_spiders(spider_id, domain, args, settings):
self.domain_crawlers.get_spider(self.spider_storage)
self.write({"status": "ok"})
else:
self.write({"status": "error"})
def on_message(self, message):
try:
msg = json.loads(message)
event, data = msg['event'], msg['data']
if event == 'rpc:request' and not checkToken(
self.key, data['params']):
self.write_event({'event': 'authority:out', 'data': None})
return
if 'token' in data['params']:
del data['params']['token']
except Exception as e:
logger.warn("Invalid message skipped" + message[:500])
return
self.handle_request(json.dumps(data))
def post(self):
if self.is_json:
if not checkToken(self.key, self.json_args):
return self.redirect("/login")
domain = self.json_args['domain']
args = self.json_args.get('options', {}).get('args', {})
settings = self.json_args.get('options', {}).get('settings', {})
if self.crawl(domain, args, settings):
self.write({
"status": "ok",
"job_id": self.crawler.spider.crawl_id
})
else:
self.write({"status": "error"})
else:
domain = self.get_body_argument('domain')
if self.crawl(domain, {}, {}):
self.redirect("/")
else:
raise HTTPError(400)
def authority_check(self, data):
if not checkToken(self.key, data):
self.write_event("authority:out", data=None)