def __init__(self, *a, **kw):
super(SitemapSpider, self).__init__(*a, **kw)
self._cbs = []
for r, c in self.sitemap_rules:
if isinstance(c, str):
c = getattr(self, c)
self._cbs.append((regex(r), c))
self._follow = [regex(x) for x in self.sitemap_follow]
def __init_sitemap_(self, sitemap_urls, custom_sitemap_rules):
"""Init method of a SiteMapSpider @Scrapy"""
self.sitemap_urls = sitemap_urls
self.sitemap_rules = custom_sitemap_rules
self._cbs = []
for r, c in self.sitemap_rules:
if isinstance(c, six.string_types):
c = getattr(self, c)
self._cbs.append((regex(r), c))
self._follow = [regex(x) for x in self.sitemap_follow]
def __init_sitemap_(self, sitemap_urls, custom_sitemap_rules):
# print "__init_sitemap_"
self.sitemap_urls = sitemap_urls
self.sitemap_rules = custom_sitemap_rules
self._cbs = []
for r, c in self.sitemap_rules:
if isinstance(c, six.string_types):
c = getattr(self, c)
self._cbs.append((regex(r), c))
self._follow = [regex(x) for x in self.sitemap_follow]
