def load_url(url, enable_cache=True, timeout=60):
start_time = clock()
cache = httplib2.FileCache(".cache")
headers = {
'Accept': 'application/samlmetadata+xml,text/xml,application/xml'
}
if not enable_cache:
headers['cache-control'] = 'no-cache'
log.debug("fetching (caching: %s) '%s'" % (enable_cache, url))
if url.startswith('file://'):
path = url[7:]
if not os.path.exists(path):
log.error("file not found: %s" % path)
return _Resource(result=None,
cached=False,
date=None,
resp=None,
time=None,
last_modified=None)
with io.open(path, 'rb') as fd:
return _Resource(result=fd.read(),
cached=False,
date=datetime.now(),
resp=None,
time=clock() - start_time,
last_modified=datetime.fromtimestamp(
os.stat(path).st_mtime))
else:
h = httplib2.Http(cache=cache,
timeout=timeout,
disable_ssl_certificate_validation=True
) # trust is done using signatures over here
log.debug("about to request %s" % url)
try:
resp, content = h.request(url, headers=headers)
except Exception as ex:
print_exc(ex)
raise ex
log.debug("got status: %d" % resp.status)
if resp.status != 200:
log.debug("got resp code %d (%d bytes)" %
(resp.status, len(content)))
raise IOError(resp.reason)
log.debug("last-modified header: %s" % resp.get('last-modified'))
log.debug("date header: %s" % resp.get('date'))
log.debug("last modified: %s" %
resp.get('date', resp.get('last-modified', None)))
return _Resource(result=content,
cached=resp.fromcache,
date=parse_date(resp['date']),
resp=resp,
time=clock() - start_time,
last_modified=parse_date(
resp.get('date', resp.get('last-modified',
None))))
def __init__(self,
cachedir=None,
auth=None,
offline_mode=False,
disable_ssl_validation=False,
log_filename=None,
timeout=None,
fail_handler=None,
extra_headers=None,
serializers=None,
default_content_type='application/json'):
"""Initialize a ``PistonRequester``.
``cachedir`` will be used as ``httplib2``'s cache directory if
provided.
``auth`` can be an instance of ``BasicAuthorizer`` or
``OAuthAuthorizer`` or any object that provides a ``sign_request``
method. If ``auth`` is ``None`` you'll only be able to make public
API calls. See :ref:`authentication` for details.
``disable_ssl_validation`` will skip server SSL certificate
validation when using secure connections. ``httplib2`` < 0.7.0
doesn't support certificate validation anyway, so if you're using an
older ``httplib2`` this will have no effect.
``offline_mode`` will not touch the network. In this case only cached
results will be available.
If you pass in a ``log_filename``, all requests and responses
including headers will be logged to this file.
"""
if cachedir:
self._create_dir_if_needed(cachedir)
self._httplib2_cache = httplib2.FileCache(cachedir, safe=safename)
else:
self._httplib2_cache = None
self._auth = auth
self._offline_mode = offline_mode
self._disable_ssl_validation = disable_ssl_validation
self._timeout = timeout
# create one httplib2.Http object per scheme so that we can
# have per-scheme proxy settings (see also Issue 26
# http://code.google.com/p/httplib2/issues/detail?id=26)
self._http = {}
for scheme in self.SUPPORTED_SCHEMAS:
self._http[scheme] = self._get_http_obj_for_scheme(scheme)
if serializers is None:
serializers = {}
self._serializers = {}
self._log_filename = log_filename
self._default_content_type = default_content_type
if fail_handler is None:
fail_handler = ExceptionFailHandler
self._fail_handler = fail_handler
self._extra_headers = extra_headers
def _get_file_cache(self, cache_dir):
cache_dir = cache_dir or _get_cache_dir()
fc = httplib2.FileCache(
cache_dir,
safe=_safecachename
)
return fc
def run(self):
def _parse_date(str):
if str is None:
return datetime.new()
return datetime(*parsedate(str)[:6])
self.start_time = clock()
try:
cache = httplib2.FileCache(".cache")
if not self.enable_cache:
log.debug("removing '%s' from cache" % self.url)
cache.delete(self.url)
log.debug("fetching '%s'" % self.url)
if self.url.startswith('file://'):
path = self.url[7:]
if not os.path.exists(path):
raise IOError("file not found: %s" % path)
with open(path, 'r') as fd:
self.result = fd.read()
self.cached = False
self.date = datetime.now()
self.last_modified = datetime.fromtimestamp(os.stat(path).st_mtime)
else:
try:
h = httplib2.Http(cache=cache, timeout=60,
disable_ssl_certificate_validation=True) # trust is done using signatures over here
resp, content = h.request(self.url)
self.status = resp.status
self.last_modified = _parse_date(resp.get('last-modified', resp.get('date', None)))
if resp.status != 200:
raise IOError(resp.reason)
self.result = content
self.cached = resp.fromcache
except Exception, ex:
resp = requests.get(self.url)
self.status = resp.status_code
self.last_modified = _parse_date(resp.headers['last-modified'] or resp.headers['date'])
if resp.status_code != 200:
raise IOError(httplib.responses[resp.status_code])
self.result = resp.content
self.cached = False
log.debug("got %d bytes from '%s'" % (len(self.result), self.url))
def load_url(url, enable_cache=True, timeout=60):
start_time = clock()
cache = httplib2.FileCache(".cache")
headers = dict()
if not enable_cache:
headers['cache-control'] = 'no-cache'
log.debug("fetching (caching: %s) '%s'" % (enable_cache, url))
if url.startswith('file://'):
path = url[7:]
if not os.path.exists(path):
raise IOError("file not found: %s" % path)
with io.open(path, 'r+b') as fd:
return _Resource(result=fd.read(),
cached=False,
date=datetime.now(),
resp=None,
time=clock() - start_time,
last_modified=datetime.fromtimestamp(
os.stat(path).st_mtime))
else:
h = httplib2.Http(cache=cache,
timeout=timeout,
disable_ssl_certificate_validation=True
) # trust is done using signatures over here
resp, content = h.request(url, headers=headers)
if resp.status != 200:
raise IOError(resp.reason)
log.debug("last-modified header: %s" % resp.get('last-modified'))
log.debug("date header: %s" % resp.get('date'))
log.debug("last modified: %s" %
resp.get('date', resp.get('last-modified', None)))
return _Resource(result=content,
cached=resp.fromcache,
date=parse_date(resp['date']),
resp=resp,
time=clock() - start_time,
last_modified=parse_date(
resp.get('date', resp.get('last-modified',
None))))
def set_http_cache(dir):
try:
cache = httplib2.FileCache(dir, safe=lambda x: md5.new(x).hexdigest())
http.cache = cache
except:
pass
def set_http_cache_dir(d):
fc = httplib2.FileCache(d)
http.cache = fc