def _fetch(self, url, form_data={}, headers={}, compression=True): ''' A wrapper around the super's _fetch with cloudflare support ''' helper.log_debug("Fetch attempt url: %s, form_data: %s, headers: %s" % (url, form_data, headers)) if not self._cloudflare: return Net._fetch(self, url, form_data, headers, compression) else: try: r = Net._fetch(self, url, form_data, headers, compression) helper.log_debug('Did not encounter a cloudflare challenge') return r except urllib2.HTTPError as e: if e.code == 503: helper.log_debug('Encountered a cloudflare challenge') challenge = e.read() if challenge == 'The service is unavailable.': helper.log_debug('Challenge says the service is unavailable') raise try: helper.log_debug("Received a challenge, so we'll need to get around cloudflare") self._resolve_cloudflare(url, challenge, form_data, headers, compression) helper.log_debug("Successfully resolved cloudflare challenge, fetching real response") return Net._fetch(self, url, form_data, headers, compression) except urllib2.HTTPError as e: helper.log_debug("Failed to set up cloudflare with exception %s" % str(e)) raise else: helper.log_debug('Initial attempt failed with code %d' % e.code) raise
def _resolve_cloudflare(self, url, challenge, form_data={}, headers={}, compression=True): """ Asks _cloudflare for an URL with the answer to overcome the challenge, and then attempts the resolution. """ helper.start("_resolve_cloudflare") parsed_url = urlparse(url) cloudflare_url = urlunparse((parsed_url.scheme, parsed_url.netloc, '', '', '', '')) query = self._get_cloudflare_answer(cloudflare_url, challenge, form_data, headers, compression) # Use the cloudflare jar instead for this attempt; revert back to # main jar after attempt with call to update_opener() self._update_opener_with_cloudflare() try: helper.log_debug("Attempting to resolve the challenge") response = Net._fetch(self, query, form_data, headers, compression) helper.log_debug("Resolved the challenge, updating cookies") for c in self._cloudflare_jar: self._cj.set_cookie(c) self._update_opener() except urllib2.HTTPError as e: helper.log_debug("Failed to resolve the cloudflare challenge with exception %s" % str(e)) self._update_opener() pass helper.end('_resolve_cloudflare')