def render_GET(self, txrequest): status = get_engine_status(self.crawler.engine) if self._spider_name is None: return status for sp, st in status['spiders'].items(): if sp.name == self._spider_name: return st
def format_engine_status_test(engine=None): checks = get_engine_status(engine) map = {} for test, result in checks: map[test] = result return map
def webconsole_render(self, wc_request): s = banner(self) s += "<pre><code>\n" s += get_engine_status() s += "</pre></code>\n" s += "</body>\n" s += "</html>\n" return s
def _send_report(self, rcpts, subject): """send notification mail with some additional useful info""" s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024) s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024) s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024) s += "ENGINE STATUS ------------------------------------------------------- \r\n" s += "\r\n" s += pformat(get_engine_status(self.crawler.engine)) s += "\r\n" self.mail.send(rcpts, subject, s)
def spider_closed(self, spider): subject = 'Scrapy Server: Spider %s closed, please check out on your server.' % spider.name content = 'Spider name: %s' % spider.name content += '\n' + '-' * 20 + '\n' content += 'Spider stats:\n' content += '\n'.join('%s: %s' % i for i in self.crawler.stats.get_stats().items()) content += '\n' + '-' * 20 + '\n' content += 'Engine status:\n' content += '\n'.join('%s: %s' % i for i in get_engine_status(self.crawler.engine)) # print(subject) print(content) send_mail(subject=subject, content=content)
def cb(response): est.append(get_engine_status(spider.crawler.engine))
def cb(response): est.append(get_engine_status(crawler.engine))
def cb(self, response): self.engine_status.append(get_engine_status(self.crawler.engine))