def get(self): jobs = self.crawler_process.jobs spiders = self.domain_crawlers.spiders initial_data_json = json.dumps( json_encode({ "jobs": jobs, "spiders": spiders })) return self.render("index.html", initial_data_json=initial_data_json)
def write_event(self, data, max_message_size=0): if isinstance(data, six.string_types): message = data else: message = json_encode(data) try: if sys.getsizeof(message) < max_message_size or not max_message_size: self.write_message(message) else: logger.info("Message size exceeded. Message wasn't sent.") except websocket.WebSocketClosedError: pass
def get(self): crawl_ids_arg = self.get_argument('crawl_ids', '') if crawl_ids_arg == '': jobs = self.crawler_process.get_jobs() else: crawl_ids = set(crawl_ids_arg.split(',')) jobs = [ job for job in self.crawler_process.get_jobs() if job['id'] in crawl_ids ] self.write(json_encode({"jobs": jobs}))
def write_event(self, event, data): """ Send a message to the client """ # if event != "process:stats": # print("-----------------================") # print("write_event {} {}".format(event, data)) message = None try: message = json_encode({'event': event, 'data': data}) except Exception as e: logger.warn("Invalid event message skipped {} {} {}".format( e, event, data)) return if message: try: self.write_message(message) except Exception as e: logger.warn("Error while sending message {}".format(e))
def get(self): jobs = self.crawler_process.jobs initial_data_json = json_encode({"jobs": jobs}) return self.render("index.html", initial_data_json=initial_data_json)
def render(self, *args, **kwargs): proc_stats = self.crawler_process.procmon.get_recent() kwargs['initial_process_stats_json'] = json_encode(proc_stats) return super(BaseRequestHandler, self).render(*args, **kwargs)
def send_data(self, data): self.write(json_encode(data)) self.finish()
def _get_stats_json(self): # json is to fix an issue with dots in key names return json_encode(self.crawler.stats.get_stats())