def __init__(self):
rs = ReadSetting() #读取各项参数
self.start_urls = rs.readurl()
self.linkmatrix = LinkMatrix(rs.projectname())
self.linkmatrix.setroot(self.start_urls)
self.allowed_domains = rs.readalloweddomain()
self.xpath = rs.readxpath()
self.rules = [Rule(LinkExtractor(), follow=True, callback="parse_start_url")]
#设置爬取规则:follow所有url;Request通过spidermiddlewares过滤掉限定域外的url;生成的response传递给parse_start_url
#所有Request均经过spidermiddlewares
super(XpathSpider, self).__init__()
def __init__(self):
rs = ReadSetting() #读取各项参数
self.start_urls = rs.readurl()
self.linkmatrix = LinkMatrix(rs.projectname())
self.linkmatrix.setroot(self.start_urls)
self.allowed_domains = rs.readalloweddomain()
self.allow, self.deny = rs.readurlmatch()
self.regex_allow = re.compile('({0})'.format('|'.join(
[re.escape(e) for e in self.allow]))) #生成正则表达式
self.regex_deny = re.compile('({0})'.format('|'.join(
[re.escape(e) for e in self.deny])))
self.rules = [
Rule(LinkExtractor(), follow=True, callback="parse_match")
]
#设置爬取规则:follow所有url;Request通过spidermiddlewares过滤掉限定域外的url;生成的response传递给parse_match
#所有Request均经过spidermiddlewares
super(MatchSpider, self).__init__()