def render_GET(self, txrequest):
status = get_engine_status(self.crawler.engine)
if self._spider_name is None:
return status
for sp, st in status['spiders'].items():
if sp.name == self._spider_name:
return st
def render_GET(self, txrequest):
status = get_engine_status(self.crawler.engine)
if self._spider_name is None:
return status
for sp, st in status['spiders'].items():
if sp.name == self._spider_name:
return st
def format_engine_status_test(engine=None):
checks = get_engine_status(engine)
map = {}
for test, result in checks:
map[test] = result
return map
def webconsole_render(self, wc_request):
s = banner(self)
s += "<pre><code>\n"
s += get_engine_status()
s += "</pre></code>\n"
s += "</body>\n"
s += "</html>\n"
return s
def _send_report(self, rcpts, subject):
"""send notification mail with some additional useful info"""
s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024)
s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024)
s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024)
s += "ENGINE STATUS ------------------------------------------------------- \r\n"
s += "\r\n"
s += pformat(get_engine_status(self.crawler.engine))
s += "\r\n"
self.mail.send(rcpts, subject, s)
def _send_report(self, rcpts, subject):
"""send notification mail with some additional useful info"""
s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024)
s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024)
s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024)
s += "ENGINE STATUS ------------------------------------------------------- \r\n"
s += "\r\n"
s += pformat(get_engine_status(self.crawler.engine))
s += "\r\n"
self.mail.send(rcpts, subject, s)
def spider_closed(self, spider):
subject = 'Scrapy Server: Spider %s closed, please check out on your server.' % spider.name
content = 'Spider name: %s' % spider.name
content += '\n' + '-' * 20 + '\n'
content += 'Spider stats:\n'
content += '\n'.join('%s: %s' % i
for i in self.crawler.stats.get_stats().items())
content += '\n' + '-' * 20 + '\n'
content += 'Engine status:\n'
content += '\n'.join('%s: %s' % i
for i in get_engine_status(self.crawler.engine))
# print(subject)
print(content)
send_mail(subject=subject, content=content)
def cb(response):
est.append(get_engine_status(spider.crawler.engine))
def cb(response):
est.append(get_engine_status(crawler.engine))
def cb(self, response):
self.engine_status.append(get_engine_status(self.crawler.engine))
def cb(self, response):
self.engine_status.append(get_engine_status(self.crawler.engine))
