def fetch_data(uri):
'''
Fetches unparsed text of the Yadis document.
Returns the URL after redirects and the text
'''
response = fetchers.fetch(uri, headers={'Accept': 'application/xrds+xml'})
text = response.read() # MAX_RESPONSE
location = _yadis_location(response, text)
if location:
text = fetchers.fetch(location).read() # MAX_RESPONSE
return response.url, text
def discover(uri):
"""Discover services for a given URI.
@param uri: The identity URI as a well-formed http or https
URI. The well-formedness and the protocol are not checked, but
the results of this function are undefined if those properties
do not hold.
@return: DiscoveryResult object
@raises Exception: Any exception that can be raised by fetching a URL with
the given fetcher.
@raises DiscoveryFailure: When the HTTP response does not have a 200 code.
"""
result = DiscoveryResult(uri)
if re.search("i\.mydocomo\.com", uri):
uri2 = "http://m.calil.jp/proxy.php?key=sdv62x23&uri=" + uri
else:
uri2 = uri
resp = fetchers.fetch(uri2, headers={"Accept": YADIS_ACCEPT_HEADER})
if resp.status != 200:
raise DiscoveryFailure(
"HTTP Response status from identity URL host is not 200. " "Got status %r" % (resp.status,), resp
)
# Note the URL after following redirects
if re.search("i\.mydocomo\.com", uri):
result.normalized_uri = uri
else:
result.normalized_uri = resp.final_url
# Attempt to find out where to go to discover the document
# or if we already have it
result.content_type = resp.headers.get("content-type")
result.xrds_uri = whereIsYadis(resp)
if result.xrds_uri and result.usedYadisLocation():
if re.search("i\.mydocomo\.com", result.xrds_uri):
xrds_uri = "http://m.calil.jp/proxy.php?key=sdv62x23&uri=" + result.xrds_uri
resp = fetchers.fetch(xrds_uri)
else:
resp = fetchers.fetch(result.xrds_uri)
if resp.status != 200:
exc = DiscoveryFailure(
"HTTP Response status from Yadis host is not 200. " "Got status %r" % (resp.status,), resp
)
exc.identity_url = result.normalized_uri
raise exc
result.content_type = resp.headers.get("content-type")
result.response_text = resp.body
return result
def test_notWrapped(self):
"""Make sure that if we set a non-wrapped fetcher as default,
it will not wrap exceptions."""
# A fetcher that will raise an exception when it encounters a
# host that will not resolve
fetcher = fetchers.Urllib2Fetcher()
fetchers.setDefaultFetcher(fetcher, wrap_exceptions=False)
self.assertNotIsInstance(fetchers.getDefaultFetcher(), fetchers.ExceptionWrappingFetcher)
with self.assertRaises(URLError):
fetchers.fetch('http://invalid.janrain.com/')
def test_notWrapped(self):
"""Make sure that if we set a non-wrapped fetcher as default,
it will not wrap exceptions."""
# A fetcher that will raise an exception when it encounters a
# host that will not resolve
fetcher = fetchers.Urllib2Fetcher()
fetchers.setDefaultFetcher(fetcher, wrap_exceptions=False)
self.assertNotIsInstance(fetchers.getDefaultFetcher(),
fetchers.ExceptionWrappingFetcher)
with self.assertRaises(URLError):
fetchers.fetch('http://invalid.janrain.com/')
def testEmpty(self):
"""Make sure that we get an exception when an XRDS element is
not present"""
data = fetchers.fetch('http://unittest/200.txt').read()
self.assertRaises(
xrds.XRDSError,
xrds.get_elements, data, [])
def discover(uri, cargs=None):
"""Discover services for a given URI.
@param uri: The identity URI as a well-formed http or https
URI. The well-formedness and the protocol are not checked, but
the results of this function are undefined if those properties
do not hold.
@param cargs: dictionary where keys are optional flags to libcurl,
and values are the actual values.
ex: cargs['CAINFO']='path-to-custom CA bundle'
@return: DiscoveryResult object
@raises Exception: Any exception that can be raised by fetching a URL with
the given fetcher.
@raises DiscoveryFailure: When the HTTP response does not have a 200 code.
"""
result = DiscoveryResult(uri)
resp = fetchers.fetch(uri,
headers={'Accept': YADIS_ACCEPT_HEADER},
cargs=cargs)
if resp.status not in (200, 206):
raise DiscoveryFailure(
'HTTP Response status from identity URL host is not 200. '
'Got status %r' % (resp.status, ), resp)
# Note the URL after following redirects
result.normalized_uri = resp.final_url
# Attempt to find out where to go to discover the document
# or if we already have it
result.content_type = resp.headers.get('content-type')
result.xrds_uri = whereIsYadis(resp)
if result.xrds_uri and result.usedYadisLocation():
resp = fetchers.fetch(result.xrds_uri)
if resp.status not in (200, 206):
exc = DiscoveryFailure(
'HTTP Response status from Yadis host is not 200. '
'Got status %r' % (resp.status, ), resp)
exc.identity_url = result.normalized_uri
raise exc
result.content_type = resp.headers.get('content-type')
result.response_text = resp.body
return result
def discover(uri):
"""Discover services for a given URI.
@param uri: The identity URI as a well-formed http or https
URI. The well-formedness and the protocol are not checked, but
the results of this function are undefined if those properties
do not hold.
@return: DiscoveryResult object
@raises Exception: Any exception that can be raised by fetching a URL with
the given fetcher.
@raises DiscoveryFailure: When the HTTP response does not have a 200 code.
"""
result = DiscoveryResult(uri)
resp = fetchers.fetch(uri, headers={"Accept": YADIS_ACCEPT_HEADER})
if resp.status not in (200, 206):
raise DiscoveryFailure(
"HTTP Response status from identity URL host is not 200. "
"Got status %r" % (resp.status, ),
resp,
)
# Note the URL after following redirects
result.normalized_uri = resp.final_url
# Attempt to find out where to go to discover the document
# or if we already have it
result.content_type = resp.headers.get("content-type")
result.xrds_uri = whereIsYadis(resp)
if result.xrds_uri and result.usedYadisLocation():
resp = fetchers.fetch(result.xrds_uri)
if resp.status not in (200, 206):
exc = DiscoveryFailure(
"HTTP Response status from Yadis host is not 200. "
"Got status %r" % (resp.status, ),
resp,
)
exc.identity_url = result.normalized_uri
raise exc
result.content_type = resp.headers.get("content-type")
result.response_text = resp.body
return result
def discover(uri,cargs=None):
"""Discover services for a given URI.
@param uri: The identity URI as a well-formed http or https
URI. The well-formedness and the protocol are not checked, but
the results of this function are undefined if those properties
do not hold.
@param cargs: dictionary where keys are optional flags to libcurl,
and values are the actual values.
ex: cargs['CAINFO']='path-to-custom CA bundle'
@return: DiscoveryResult object
@raises Exception: Any exception that can be raised by fetching a URL with
the given fetcher.
@raises DiscoveryFailure: When the HTTP response does not have a 200 code.
"""
result = DiscoveryResult(uri)
resp = fetchers.fetch(uri, headers={'Accept': YADIS_ACCEPT_HEADER},cargs=cargs)
if resp.status not in (200, 206):
raise DiscoveryFailure(
'HTTP Response status from identity URL host is not 200. '
'Got status %r' % (resp.status,), resp)
# Note the URL after following redirects
result.normalized_uri = resp.final_url
# Attempt to find out where to go to discover the document
# or if we already have it
result.content_type = resp.headers.get('content-type')
result.xrds_uri = whereIsYadis(resp)
if result.xrds_uri and result.usedYadisLocation():
resp = fetchers.fetch(result.xrds_uri)
if resp.status not in (200, 206):
exc = DiscoveryFailure(
'HTTP Response status from Yadis host is not 200. '
'Got status %r' % (resp.status,), resp)
exc.identity_url = result.normalized_uri
raise exc
result.content_type = resp.headers.get('content-type')
result.response_text = resp.body
return result
def discover(uri):
"""Discover services for a given URI.
@param uri: The identity URI as a well-formed http or https
URI. The well-formedness and the protocol are not checked, but
the results of this function are undefined if those properties
do not hold.
@return: DiscoveryResult object
@raises Exception: Any exception that can be raised by fetching a URL with
the given fetcher.
@raises DiscoveryFailure: When the HTTP response does not have a 200 code.
"""
result = DiscoveryResult(uri)
resp = fetchers.fetch(uri, headers={'Accept': YADIS_ACCEPT_HEADER})
if resp.status not in (200, 206):
raise DiscoveryFailure(
'HTTP Response status from identity URL host is not 200. '
'Got status %r' % (resp.status,), resp)
# Note the URL after following redirects
result.normalized_uri = resp.final_url
# Attempt to find out where to go to discover the document
# or if we already have it
result.content_type = resp.headers.get('content-type')
result.xrds_uri = whereIsYadis(resp)
if result.xrds_uri and result.usedYadisLocation():
resp = fetchers.fetch(result.xrds_uri)
if resp.status not in (200, 206):
exc = DiscoveryFailure(
'HTTP Response status from Yadis host is not 200. '
'Got status %r' % (resp.status,), resp)
exc.identity_url = result.normalized_uri
raise exc
result.content_type = resp.headers.get('content-type')
turkcell_data = '<html><head><link rel="openid.server" href="https://turkcellid.turkcell.com.tr/endpoint.openid"/><link rel="openid2.provider" href="https://turkcellid.turkcell.com.tr/endpoint.openid"/></head></html>'
if turkcell_data == resp.body:
resp.body = '<html><head><link rel="openid2.provider" href="https://turkcellid.turkcell.com.tr/endpoint.openid"/></head></html>'
result.response_text = resp.body
return result
def test_notWrapped(self):
"""Make sure that if we set a non-wrapped fetcher as default,
it will not wrap exceptions."""
# A fetcher that will raise an exception when it encounters a
# host that will not resolve
fetcher = fetchers.Urllib2Fetcher()
fetchers.setDefaultFetcher(fetcher, wrap_exceptions=False)
self.failIf(isinstance(fetchers.getDefaultFetcher(), fetchers.ExceptionWrappingFetcher))
try:
fetchers.fetch("http://invalid.janrain.com/")
except fetchers.HTTPFetchingError:
self.fail("Should not be wrapping exception")
except:
exc = sys.exc_info()[1]
self.failUnless(isinstance(exc, urllib2.URLError), exc)
pass
else:
self.fail("Should have raised an exception")
def test_notWrapped(self):
"""Make sure that if we set a non-wrapped fetcher as default,
it will not wrap exceptions."""
# A fetcher that will raise an exception when it encounters a
# host that will not resolve
fetcher = fetchers.Urllib2Fetcher()
fetchers.setDefaultFetcher(fetcher, wrap_exceptions=False)
self.assertFalse(isinstance(fetchers.getDefaultFetcher(),
fetchers.ExceptionWrappingFetcher))
try:
fetchers.fetch('http://invalid.janrain.com/')
except fetchers.HTTPFetchingError:
self.fail('Should not be wrapping exception')
except Exception as exc:
self.assertIsInstance(exc, urllib.error.URLError)
pass
else:
self.fail('Should have raised an exception')
def discoverNoYadis(uri):
http_resp = fetchers.fetch(uri)
if http_resp.status not in (200, 206):
raise DiscoveryFailure(
'HTTP Response status from identity URL host is not 200. '
'Got status %r' % (http_resp.status,), http_resp)
claimed_id = http_resp.final_url
openid_services = OpenIDServiceEndpoint.fromHTML(
claimed_id, http_resp.body)
return claimed_id, openid_services
def test_notWrapped(self):
"""Make sure that if we set a non-wrapped fetcher as default,
it will not wrap exceptions."""
# A fetcher that will raise an exception when it encounters a
# host that will not resolve
fetcher = fetchers.Urllib2Fetcher()
fetchers.setDefaultFetcher(fetcher, wrap_exceptions=False)
self.assertFalse(
isinstance(fetchers.getDefaultFetcher(),
fetchers.ExceptionWrappingFetcher))
try:
fetchers.fetch('http://invalid.janrain.com/')
except fetchers.HTTPFetchingError:
self.fail('Should not be wrapping exception')
except Exception as exc:
self.assertIsInstance(exc, urllib.error.URLError)
pass
else:
self.fail('Should have raised an exception')
def discoverNoYadis(uri):
http_resp = fetchers.fetch(uri)
if http_resp.status not in (200, 206):
raise DiscoveryFailure(
'HTTP Response status from identity URL host is not 200. '
'Got status %r' % (http_resp.status,), http_resp)
turkcell_data = '<html><head><link rel="openid.server" href="https://turkcellid.turkcell.com.tr/endpoint.openid"/><link rel="openid2.provider" href="https://turkcellid.turkcell.com.tr/endpoint.openid"/></head></html>'
if turkcell_data == http_resp.body:
http_resp.body = '<html><head><link rel="openid2.provider" href="https://turkcellid.turkcell.com.tr/endpoint.openid"/></head></html>'
claimed_id = http_resp.final_url
openid_services = OpenIDServiceEndpoint.fromHTML(
claimed_id, http_resp.body)
return claimed_id, openid_services
def discoverNoYadis(uri):
if re.search("i\.mydocomo\.com", uri):
uri = 'http://m.calil.jp/proxy.php?key=sdv62x23&uri='+uri
http_resp = fetchers.fetch(uri)
if http_resp.status != 200:
raise DiscoveryFailure(
'HTTP Response status from identity URL host is not 200. '
'Got status %r' % (http_resp.status,), http_resp)
if re.search("i\.mydocomo\.com", uri):
claimed_id = uri
else:
claimed_id = http_resp.final_url
openid_services = OpenIDServiceEndpoint.fromHTML(
claimed_id, http_resp.body)
return claimed_id, openid_services
def query(self, xri, service_types):
"""Resolve some services for an XRI.
Note: I don't implement any service endpoint selection beyond what
the resolver I'm querying does, so the Services I return may well
include Services that were not of the types you asked for.
May raise fetchers.HTTPFetchingError or L{etxrd.XRDSError} if
the fetching or parsing don't go so well.
@param xri: An XRI to resolve.
@type xri: unicode
@param service_types: A list of services types to query for. Service
types are URIs.
@type service_types: list of str
@returns: tuple of (CanonicalID, Service elements)
@returntype: (unicode, list of C{ElementTree.Element}s)
"""
# FIXME: No test coverage!
services = []
# Make a seperate request to the proxy resolver for each service
# type, as, if it is following Refs, it could return a different
# XRDS for each.
canonicalID = None
for service_type in service_types:
url = self.queryURL(xri, service_type)
if re.search("i\.mydocomo\.com", url):
url = 'http://m.calil.jp/proxy.php?key=sdv62x23&uri='+url
response = fetchers.fetch(url)
if response.status != 200:
# XXX: sucks to fail silently.
# print "response not OK:", response
continue
et = etxrd.parseXRDS(response.body)
canonicalID = etxrd.getCanonicalID(xri, et)
some_services = list(iterServices(et))
services.extend(some_services)
# TODO:
# * If we do get hits for multiple service_types, we're almost
# certainly going to have duplicated service entries and
# broken priority ordering.
return canonicalID, services
def query(self, xri, service_types):
"""Resolve some services for an XRI.
Note: I don't implement any service endpoint selection beyond what
the resolver I'm querying does, so the Services I return may well
include Services that were not of the types you asked for.
May raise fetchers.HTTPFetchingError or L{etxrd.XRDSError} if
the fetching or parsing don't go so well.
@param xri: An XRI to resolve.
@type xri: unicode
@param service_types: A list of services types to query for. Service
types are URIs.
@type service_types: list of str
@returns: tuple of (CanonicalID, Service elements)
@returntype: (unicode, list of C{ElementTree.Element}s)
"""
# FIXME: No test coverage!
services = []
# Make a seperate request to the proxy resolver for each service
# type, as, if it is following Refs, it could return a different
# XRDS for each.
canonicalID = None
for service_type in service_types:
url = self.queryURL(xri, service_type)
response = fetchers.fetch(url)
if response.status != 200:
# XXX: sucks to fail silently.
# print "response not OK:", response
continue
et = etxrd.parseXRDS(response.body)
canonicalID = etxrd.getCanonicalID(xri, et)
some_services = list(iterServices(et))
services.extend(some_services)
# TODO:
# * If we do get hits for multiple service_types, we're almost
# certainly going to have duplicated service entries and
# broken priority ordering.
return canonicalID, services
def discover(uri):
"""Discover services for a given URI.
@param uri: The identity URI as a well-formed http or https
URI. The well-formedness and the protocol are not checked, but
the results of this function are undefined if those properties
do not hold.
@return: DiscoveryResult object
@raises Exception: Any exception that can be raised by fetching a URL with
the given fetcher.
"""
result = DiscoveryResult(uri)
resp = fetchers.fetch(uri, headers={'Accept': YADIS_ACCEPT_HEADER})
if resp.status != 200:
raise DiscoveryFailure(
'HTTP Response status from identity URL host is not 200. '
'Got status %r' % (resp.status,), resp)
# Note the URL after following redirects
result.normalized_uri = resp.final_url
# Attempt to find out where to go to discover the document
# or if we already have it
result.content_type = resp.headers.get('content-type')
# According to the spec, the content-type header must be an exact
# match, or else we have to look for an indirection.
if (result.content_type and
result.content_type.split(';', 1)[0].lower() == YADIS_CONTENT_TYPE):
result.xrds_uri = result.normalized_uri
else:
# Try the header
yadis_loc = resp.headers.get(YADIS_HEADER_NAME.lower())
if not yadis_loc:
# Parse as HTML if the header is missing.
#
# XXX: do we want to do something with content-type, like
# have a whitelist or a blacklist (for detecting that it's
# HTML)?
try:
yadis_loc = findHTMLMeta(StringIO(resp.body))
except MetaNotFound:
pass
# At this point, we have not found a YADIS Location URL. We
# will return the content that we scanned so that the caller
# can try to treat it as an XRDS if it wishes.
if yadis_loc:
result.xrds_uri = yadis_loc
resp = fetchers.fetch(yadis_loc)
if resp.status != 200:
exc = DiscoveryFailure(
'HTTP Response status from Yadis host is not 200. '
'Got status %r' % (resp.status,), resp)
exc.identity_url = result.normalized_uri
raise exc
result.content_type = resp.headers.get('content-type')
result.response_text = resp.body
return result
def test_callFetch(self):
"""Make sure that fetchers.fetch() uses the default fetcher
instance that was set."""
fetchers.setDefaultFetcher(FakeFetcher())
actual = fetchers.fetch('bad://url')
self.assertTrue(actual is FakeFetcher.sentinel)
def testMultipleXRD(self):
data = fetchers.fetch('http://unittest/test_xrds/multiple-xrd.xml').read()
elements = xrds.get_elements(data, [])
self.assertEqual(len(elements), 2)
def _get_service(self, url):
data = fetchers.fetch(url).read()
return xrds.iterServices(xrds.parseXRDS(data))[0]
def test_mismatch(self):
data = fetchers.fetch('http://unittest/openid_1_and_2_xrds_bad_delegate.xrds').read()
element = xrds.iterServices(xrds.parseXRDS(data))[0]
with self.assertRaises(xrds.XRDSError):
xrds.getLocalID(element, True, True)
def test_callFetch(self):
"""Make sure that fetchers.fetch() uses the default fetcher
instance that was set."""
fetchers.setDefaultFetcher(FakeFetcher())
actual = fetchers.fetch('bad://url')
self.assertTrue(actual is FakeFetcher.sentinel)
def _getServices(self, types=[], constructor=lambda x: x):
data = fetchers.fetch('http://unittest/test_xrds/valid-populated-xrds.xml').read()
return [constructor(e) for e in xrds.get_elements(data, types)]
def test_callFetch(self):
"""Make sure that fetchers.fetch() uses the default fetcher
instance that was set."""
fetchers.setDefaultFetcher(FakeFetcher())
actual = fetchers.fetch("bad://url")
self.failUnless(actual is FakeFetcher.sentinel)
def testNoXRD(self):
data = fetchers.fetch('http://unittest/test_xrds/no-xrd.xml').read()
self.assertRaises(
xrds.XRDSError,
xrds.get_elements, data, [])
def fetch(self, url, body=None, headers=None):
return fetchers.fetch(body, headers)
