def __init__(self): rs = ReadSetting() #读取各项参数 self.start_urls = rs.readurl() self.linkmatrix = LinkMatrix(rs.projectname()) self.linkmatrix.setroot(self.start_urls) self.allowed_domains = rs.readalloweddomain() self.xpath = rs.readxpath() self.rules = [Rule(LinkExtractor(), follow=True, callback="parse_start_url")] #设置爬取规则:follow所有url;Request通过spidermiddlewares过滤掉限定域外的url;生成的response传递给parse_start_url #所有Request均经过spidermiddlewares super(XpathSpider, self).__init__()
def __init__(self): rs = ReadSetting() #读取各项参数 self.start_urls = rs.readurl() self.linkmatrix = LinkMatrix(rs.projectname()) self.linkmatrix.setroot(self.start_urls) self.allowed_domains = rs.readalloweddomain() self.allow, self.deny = rs.readurlmatch() self.regex_allow = re.compile('({0})'.format('|'.join( [re.escape(e) for e in self.allow]))) #生成正则表达式 self.regex_deny = re.compile('({0})'.format('|'.join( [re.escape(e) for e in self.deny]))) self.rules = [ Rule(LinkExtractor(), follow=True, callback="parse_match") ] #设置爬取规则:follow所有url;Request通过spidermiddlewares过滤掉限定域外的url;生成的response传递给parse_match #所有Request均经过spidermiddlewares super(MatchSpider, self).__init__()