def _fetch(self, url, form_data={}, headers={}, compression=True):
'''
A wrapper around the super's _fetch with cloudflare support
'''
helper.log_debug("Fetch attempt url: %s, form_data: %s, headers: %s" % (url, form_data, headers))
if not self._cloudflare:
return Net._fetch(self, url, form_data, headers, compression)
else:
try:
r = Net._fetch(self, url, form_data, headers, compression)
helper.log_debug('Did not encounter a cloudflare challenge')
return r
except urllib2.HTTPError as e:
if e.code == 503:
helper.log_debug('Encountered a cloudflare challenge')
challenge = e.read()
if challenge == 'The service is unavailable.':
helper.log_debug('Challenge says the service is unavailable')
raise
try:
helper.log_debug("Received a challenge, so we'll need to get around cloudflare")
self._resolve_cloudflare(url, challenge, form_data, headers, compression)
helper.log_debug("Successfully resolved cloudflare challenge, fetching real response")
return Net._fetch(self, url, form_data, headers, compression)
except urllib2.HTTPError as e:
helper.log_debug("Failed to set up cloudflare with exception %s" % str(e))
raise
else:
helper.log_debug('Initial attempt failed with code %d' % e.code)
raise
def _resolve_cloudflare(self, url, challenge, form_data={}, headers={}, compression=True):
"""
Asks _cloudflare for an URL with the answer to overcome the
challenge, and then attempts the resolution.
"""
helper.start("_resolve_cloudflare")
parsed_url = urlparse(url)
cloudflare_url = urlunparse((parsed_url.scheme, parsed_url.netloc, '', '', '', ''))
query = self._get_cloudflare_answer(cloudflare_url, challenge, form_data, headers, compression)
# Use the cloudflare jar instead for this attempt; revert back to
# main jar after attempt with call to update_opener()
self._update_opener_with_cloudflare()
try:
helper.log_debug("Attempting to resolve the challenge")
response = Net._fetch(self, query, form_data, headers, compression)
helper.log_debug("Resolved the challenge, updating cookies")
for c in self._cloudflare_jar:
self._cj.set_cookie(c)
self._update_opener()
except urllib2.HTTPError as e:
helper.log_debug("Failed to resolve the cloudflare challenge with exception %s" % str(e))
self._update_opener()
pass
helper.end('_resolve_cloudflare')
