def error(failure):
"""
爬取错误的时候发送邮件
:param failure:
:return:
"""
params = {"finishTime": datetime.datetime.now(), "failure": failure}
subject = "爬虫结束状态汇报：name = baidu, finishedTime = {finishTime}".format(finishTime=datetime.datetime.now())
body = "细节：failure = {failure}, failure! at:{finishTime}".format(**params)
emailSender.EmailSender().send_email(subject=subject, body=body) # 发送邮件
def closed(reason):
"""
爬取结束的时候发送邮件
:param reason:
:return:
"""
params = {"finishTime": datetime.datetime.now(), "reason": reason}
subject = "爬虫结束状态汇报：name = baidu, finishedTime = {finishTime}".format(finishTime=datetime.datetime.now())
body = "细节：reason = {reason}, successs! at:{finishTime}".format(**params)
emailSender.EmailSender().send_email(subject=subject, body=body) # 发送邮件
def start_requests(self):
"""
1、爬虫启动时发送邮件
2、遍历分页
:return:
"""
params = {"startTime": datetime.datetime.now(), "name": self.name}
subject = "爬虫启动状态汇报：name = {name}, startTime = {startTime}".format(**params)
body = "细节：start successs! name = {name},at:{startTime}".format(**params)
emailSender.EmailSender().send_email(subject=subject, body=body) # 发送邮件
yield scrapy.Request(url=self.start_urls[0], callback=self.parse) # 首页
def start_requests(self):
"""
1、爬虫启动时发送邮件
2、遍历分页
:return:
"""
params = {"startTime": datetime.datetime.now(), "name": self.name}
subject = "爬虫启动状态汇报：name = {name}, startTime = {startTime}".format(**params)
body = "细节：start successs! name = {name},at:{startTime}".format(**params)
emailSender.EmailSender().send_email(subject=subject, body=body) # 发送邮件
for i in range(0, 45):
paging_url = self.url_model.format(page_num=str(i + 1))
meta = {"paging_url": paging_url}
yield scrapy.Request(url=paging_url, callback=self.parse, meta=meta)