def get(self):
jobs = self.crawler_process.jobs
spiders = self.domain_crawlers.spiders
initial_data_json = json.dumps(
json_encode({
"jobs": jobs,
"spiders": spiders
}))
return self.render("index.html", initial_data_json=initial_data_json)
def write_event(self, data, max_message_size=0):
if isinstance(data, six.string_types):
message = data
else:
message = json_encode(data)
try:
if sys.getsizeof(message) < max_message_size or not max_message_size:
self.write_message(message)
else:
logger.info("Message size exceeded. Message wasn't sent.")
except websocket.WebSocketClosedError:
pass
def get(self):
crawl_ids_arg = self.get_argument('crawl_ids', '')
if crawl_ids_arg == '':
jobs = self.crawler_process.get_jobs()
else:
crawl_ids = set(crawl_ids_arg.split(','))
jobs = [
job for job in self.crawler_process.get_jobs()
if job['id'] in crawl_ids
]
self.write(json_encode({"jobs": jobs}))
def write_event(self, event, data):
""" Send a message to the client """
# if event != "process:stats":
# print("-----------------================")
# print("write_event {} {}".format(event, data))
message = None
try:
message = json_encode({'event': event, 'data': data})
except Exception as e:
logger.warn("Invalid event message skipped {} {} {}".format(
e, event, data))
return
if message:
try:
self.write_message(message)
except Exception as e:
logger.warn("Error while sending message {}".format(e))
def get(self):
jobs = self.crawler_process.jobs
initial_data_json = json_encode({"jobs": jobs})
return self.render("index.html", initial_data_json=initial_data_json)
def render(self, *args, **kwargs):
proc_stats = self.crawler_process.procmon.get_recent()
kwargs['initial_process_stats_json'] = json_encode(proc_stats)
return super(BaseRequestHandler, self).render(*args, **kwargs)
def send_data(self, data):
self.write(json_encode(data))
self.finish()
def _get_stats_json(self):
# json is to fix an issue with dots in key names
return json_encode(self.crawler.stats.get_stats())