def post(self): if self.is_json: if not checkToken(self.key, self.json_args): return self.redirect("/login") job_id = self.json_args['job_id'] self.control_job(job_id) self.write({"status": "ok"}) else: data = {} data['token'] = self.get_body_argument('token') if not checkToken(self.key, data): return self.redirect("/login") job_id = self.get_body_argument('job_id') self.control_job(job_id) self.redirect("/")
def post(self): if self.is_json: if not checkToken(self.key, self.json_args): return self.redirect("/login") spider_id = self.json_args.get('_id', None) if self.remove_spiders(spider_id): self.domain_crawlers.get_spider(self.spider_storage) self.write({"status": "ok"}) else: self.write({"status": "error"})
def post(self): if self.is_json: if not checkToken(self.key, self.json_args): return self.redirect("/login") spider_id = self.json_args.get('_id', None) domain = self.json_args['domain'] args = self.json_args.get('options', {}).get('args', {}) settings = self.json_args.get('options', {}).get('settings', {}) if self.save_spiders(spider_id, domain, args, settings): self.domain_crawlers.get_spider(self.spider_storage) self.write({"status": "ok"}) else: self.write({"status": "error"})
def on_message(self, message): try: msg = json.loads(message) event, data = msg['event'], msg['data'] if event == 'rpc:request' and not checkToken( self.key, data['params']): self.write_event({'event': 'authority:out', 'data': None}) return if 'token' in data['params']: del data['params']['token'] except Exception as e: logger.warn("Invalid message skipped" + message[:500]) return self.handle_request(json.dumps(data))
def post(self): if self.is_json: if not checkToken(self.key, self.json_args): return self.redirect("/login") domain = self.json_args['domain'] args = self.json_args.get('options', {}).get('args', {}) settings = self.json_args.get('options', {}).get('settings', {}) if self.crawl(domain, args, settings): self.write({ "status": "ok", "job_id": self.crawler.spider.crawl_id }) else: self.write({"status": "error"}) else: domain = self.get_body_argument('domain') if self.crawl(domain, {}, {}): self.redirect("/") else: raise HTTPError(400)
def authority_check(self, data): if not checkToken(self.key, data): self.write_event("authority:out", data=None)