def __init__(self, *a, **kw): super(SitemapSpider, self).__init__(*a, **kw) self._cbs = [] for r, c in self.sitemap_rules: if isinstance(c, str): c = getattr(self, c) self._cbs.append((regex(r), c)) self._follow = [regex(x) for x in self.sitemap_follow]
def __init_sitemap_(self, sitemap_urls, custom_sitemap_rules): """Init method of a SiteMapSpider @Scrapy""" self.sitemap_urls = sitemap_urls self.sitemap_rules = custom_sitemap_rules self._cbs = [] for r, c in self.sitemap_rules: if isinstance(c, six.string_types): c = getattr(self, c) self._cbs.append((regex(r), c)) self._follow = [regex(x) for x in self.sitemap_follow]
def __init_sitemap_(self, sitemap_urls, custom_sitemap_rules): # print "__init_sitemap_" self.sitemap_urls = sitemap_urls self.sitemap_rules = custom_sitemap_rules self._cbs = [] for r, c in self.sitemap_rules: if isinstance(c, six.string_types): c = getattr(self, c) self._cbs.append((regex(r), c)) self._follow = [regex(x) for x in self.sitemap_follow]