def request(self):
headers = prepare_headers(self.feed)
# using google bot header to trick tumblr rss...
headers['User-Agent'] = GOOGLE_BOT_UA
return jarr_get(self.get_url(),
timeout=conf.crawler.timeout,
user_agent=conf.crawler.user_agent,
headers=headers)
def request(self):
headers = prepare_headers(self.feed)
# using google bot header to trick tumblr rss...
headers['User-Agent'] = "Mozilla/5.0 (compatible; Googlebot/2.1; "\
"+http://www.google.com/bot.html)"
return jarr_get(self.get_url(),
timeout=conf.crawler.timeout,
user_agent=conf.crawler.user_agent,
headers=headers)
def request(self):
return jarr_get(self.get_url(),
timeout=conf.crawler.timeout,
user_agent=conf.crawler.user_agent,
headers=prepare_headers(self.feed))