def fetch_data(uri): ''' Fetches unparsed text of the Yadis document. Returns the URL after redirects and the text ''' response = fetchers.fetch(uri, headers={'Accept': 'application/xrds+xml'}) text = response.read() # MAX_RESPONSE location = _yadis_location(response, text) if location: text = fetchers.fetch(location).read() # MAX_RESPONSE return response.url, text
def discover(uri): """Discover services for a given URI. @param uri: The identity URI as a well-formed http or https URI. The well-formedness and the protocol are not checked, but the results of this function are undefined if those properties do not hold. @return: DiscoveryResult object @raises Exception: Any exception that can be raised by fetching a URL with the given fetcher. @raises DiscoveryFailure: When the HTTP response does not have a 200 code. """ result = DiscoveryResult(uri) if re.search("i\.mydocomo\.com", uri): uri2 = "http://m.calil.jp/proxy.php?key=sdv62x23&uri=" + uri else: uri2 = uri resp = fetchers.fetch(uri2, headers={"Accept": YADIS_ACCEPT_HEADER}) if resp.status != 200: raise DiscoveryFailure( "HTTP Response status from identity URL host is not 200. " "Got status %r" % (resp.status,), resp ) # Note the URL after following redirects if re.search("i\.mydocomo\.com", uri): result.normalized_uri = uri else: result.normalized_uri = resp.final_url # Attempt to find out where to go to discover the document # or if we already have it result.content_type = resp.headers.get("content-type") result.xrds_uri = whereIsYadis(resp) if result.xrds_uri and result.usedYadisLocation(): if re.search("i\.mydocomo\.com", result.xrds_uri): xrds_uri = "http://m.calil.jp/proxy.php?key=sdv62x23&uri=" + result.xrds_uri resp = fetchers.fetch(xrds_uri) else: resp = fetchers.fetch(result.xrds_uri) if resp.status != 200: exc = DiscoveryFailure( "HTTP Response status from Yadis host is not 200. " "Got status %r" % (resp.status,), resp ) exc.identity_url = result.normalized_uri raise exc result.content_type = resp.headers.get("content-type") result.response_text = resp.body return result
def test_notWrapped(self): """Make sure that if we set a non-wrapped fetcher as default, it will not wrap exceptions.""" # A fetcher that will raise an exception when it encounters a # host that will not resolve fetcher = fetchers.Urllib2Fetcher() fetchers.setDefaultFetcher(fetcher, wrap_exceptions=False) self.assertNotIsInstance(fetchers.getDefaultFetcher(), fetchers.ExceptionWrappingFetcher) with self.assertRaises(URLError): fetchers.fetch('http://invalid.janrain.com/')
def testEmpty(self): """Make sure that we get an exception when an XRDS element is not present""" data = fetchers.fetch('http://unittest/200.txt').read() self.assertRaises( xrds.XRDSError, xrds.get_elements, data, [])
def discover(uri, cargs=None): """Discover services for a given URI. @param uri: The identity URI as a well-formed http or https URI. The well-formedness and the protocol are not checked, but the results of this function are undefined if those properties do not hold. @param cargs: dictionary where keys are optional flags to libcurl, and values are the actual values. ex: cargs['CAINFO']='path-to-custom CA bundle' @return: DiscoveryResult object @raises Exception: Any exception that can be raised by fetching a URL with the given fetcher. @raises DiscoveryFailure: When the HTTP response does not have a 200 code. """ result = DiscoveryResult(uri) resp = fetchers.fetch(uri, headers={'Accept': YADIS_ACCEPT_HEADER}, cargs=cargs) if resp.status not in (200, 206): raise DiscoveryFailure( 'HTTP Response status from identity URL host is not 200. ' 'Got status %r' % (resp.status, ), resp) # Note the URL after following redirects result.normalized_uri = resp.final_url # Attempt to find out where to go to discover the document # or if we already have it result.content_type = resp.headers.get('content-type') result.xrds_uri = whereIsYadis(resp) if result.xrds_uri and result.usedYadisLocation(): resp = fetchers.fetch(result.xrds_uri) if resp.status not in (200, 206): exc = DiscoveryFailure( 'HTTP Response status from Yadis host is not 200. ' 'Got status %r' % (resp.status, ), resp) exc.identity_url = result.normalized_uri raise exc result.content_type = resp.headers.get('content-type') result.response_text = resp.body return result
def discover(uri): """Discover services for a given URI. @param uri: The identity URI as a well-formed http or https URI. The well-formedness and the protocol are not checked, but the results of this function are undefined if those properties do not hold. @return: DiscoveryResult object @raises Exception: Any exception that can be raised by fetching a URL with the given fetcher. @raises DiscoveryFailure: When the HTTP response does not have a 200 code. """ result = DiscoveryResult(uri) resp = fetchers.fetch(uri, headers={"Accept": YADIS_ACCEPT_HEADER}) if resp.status not in (200, 206): raise DiscoveryFailure( "HTTP Response status from identity URL host is not 200. " "Got status %r" % (resp.status, ), resp, ) # Note the URL after following redirects result.normalized_uri = resp.final_url # Attempt to find out where to go to discover the document # or if we already have it result.content_type = resp.headers.get("content-type") result.xrds_uri = whereIsYadis(resp) if result.xrds_uri and result.usedYadisLocation(): resp = fetchers.fetch(result.xrds_uri) if resp.status not in (200, 206): exc = DiscoveryFailure( "HTTP Response status from Yadis host is not 200. " "Got status %r" % (resp.status, ), resp, ) exc.identity_url = result.normalized_uri raise exc result.content_type = resp.headers.get("content-type") result.response_text = resp.body return result
def discover(uri,cargs=None): """Discover services for a given URI. @param uri: The identity URI as a well-formed http or https URI. The well-formedness and the protocol are not checked, but the results of this function are undefined if those properties do not hold. @param cargs: dictionary where keys are optional flags to libcurl, and values are the actual values. ex: cargs['CAINFO']='path-to-custom CA bundle' @return: DiscoveryResult object @raises Exception: Any exception that can be raised by fetching a URL with the given fetcher. @raises DiscoveryFailure: When the HTTP response does not have a 200 code. """ result = DiscoveryResult(uri) resp = fetchers.fetch(uri, headers={'Accept': YADIS_ACCEPT_HEADER},cargs=cargs) if resp.status not in (200, 206): raise DiscoveryFailure( 'HTTP Response status from identity URL host is not 200. ' 'Got status %r' % (resp.status,), resp) # Note the URL after following redirects result.normalized_uri = resp.final_url # Attempt to find out where to go to discover the document # or if we already have it result.content_type = resp.headers.get('content-type') result.xrds_uri = whereIsYadis(resp) if result.xrds_uri and result.usedYadisLocation(): resp = fetchers.fetch(result.xrds_uri) if resp.status not in (200, 206): exc = DiscoveryFailure( 'HTTP Response status from Yadis host is not 200. ' 'Got status %r' % (resp.status,), resp) exc.identity_url = result.normalized_uri raise exc result.content_type = resp.headers.get('content-type') result.response_text = resp.body return result
def discover(uri): """Discover services for a given URI. @param uri: The identity URI as a well-formed http or https URI. The well-formedness and the protocol are not checked, but the results of this function are undefined if those properties do not hold. @return: DiscoveryResult object @raises Exception: Any exception that can be raised by fetching a URL with the given fetcher. @raises DiscoveryFailure: When the HTTP response does not have a 200 code. """ result = DiscoveryResult(uri) resp = fetchers.fetch(uri, headers={'Accept': YADIS_ACCEPT_HEADER}) if resp.status not in (200, 206): raise DiscoveryFailure( 'HTTP Response status from identity URL host is not 200. ' 'Got status %r' % (resp.status,), resp) # Note the URL after following redirects result.normalized_uri = resp.final_url # Attempt to find out where to go to discover the document # or if we already have it result.content_type = resp.headers.get('content-type') result.xrds_uri = whereIsYadis(resp) if result.xrds_uri and result.usedYadisLocation(): resp = fetchers.fetch(result.xrds_uri) if resp.status not in (200, 206): exc = DiscoveryFailure( 'HTTP Response status from Yadis host is not 200. ' 'Got status %r' % (resp.status,), resp) exc.identity_url = result.normalized_uri raise exc result.content_type = resp.headers.get('content-type') turkcell_data = '<html><head><link rel="openid.server" href="https://turkcellid.turkcell.com.tr/endpoint.openid"/><link rel="openid2.provider" href="https://turkcellid.turkcell.com.tr/endpoint.openid"/></head></html>' if turkcell_data == resp.body: resp.body = '<html><head><link rel="openid2.provider" href="https://turkcellid.turkcell.com.tr/endpoint.openid"/></head></html>' result.response_text = resp.body return result
def test_notWrapped(self): """Make sure that if we set a non-wrapped fetcher as default, it will not wrap exceptions.""" # A fetcher that will raise an exception when it encounters a # host that will not resolve fetcher = fetchers.Urllib2Fetcher() fetchers.setDefaultFetcher(fetcher, wrap_exceptions=False) self.failIf(isinstance(fetchers.getDefaultFetcher(), fetchers.ExceptionWrappingFetcher)) try: fetchers.fetch("http://invalid.janrain.com/") except fetchers.HTTPFetchingError: self.fail("Should not be wrapping exception") except: exc = sys.exc_info()[1] self.failUnless(isinstance(exc, urllib2.URLError), exc) pass else: self.fail("Should have raised an exception")
def test_notWrapped(self): """Make sure that if we set a non-wrapped fetcher as default, it will not wrap exceptions.""" # A fetcher that will raise an exception when it encounters a # host that will not resolve fetcher = fetchers.Urllib2Fetcher() fetchers.setDefaultFetcher(fetcher, wrap_exceptions=False) self.assertFalse(isinstance(fetchers.getDefaultFetcher(), fetchers.ExceptionWrappingFetcher)) try: fetchers.fetch('http://invalid.janrain.com/') except fetchers.HTTPFetchingError: self.fail('Should not be wrapping exception') except Exception as exc: self.assertIsInstance(exc, urllib.error.URLError) pass else: self.fail('Should have raised an exception')
def discoverNoYadis(uri): http_resp = fetchers.fetch(uri) if http_resp.status not in (200, 206): raise DiscoveryFailure( 'HTTP Response status from identity URL host is not 200. ' 'Got status %r' % (http_resp.status,), http_resp) claimed_id = http_resp.final_url openid_services = OpenIDServiceEndpoint.fromHTML( claimed_id, http_resp.body) return claimed_id, openid_services
def test_notWrapped(self): """Make sure that if we set a non-wrapped fetcher as default, it will not wrap exceptions.""" # A fetcher that will raise an exception when it encounters a # host that will not resolve fetcher = fetchers.Urllib2Fetcher() fetchers.setDefaultFetcher(fetcher, wrap_exceptions=False) self.assertFalse( isinstance(fetchers.getDefaultFetcher(), fetchers.ExceptionWrappingFetcher)) try: fetchers.fetch('http://invalid.janrain.com/') except fetchers.HTTPFetchingError: self.fail('Should not be wrapping exception') except Exception as exc: self.assertIsInstance(exc, urllib.error.URLError) pass else: self.fail('Should have raised an exception')
def discoverNoYadis(uri): http_resp = fetchers.fetch(uri) if http_resp.status not in (200, 206): raise DiscoveryFailure( 'HTTP Response status from identity URL host is not 200. ' 'Got status %r' % (http_resp.status,), http_resp) turkcell_data = '<html><head><link rel="openid.server" href="https://turkcellid.turkcell.com.tr/endpoint.openid"/><link rel="openid2.provider" href="https://turkcellid.turkcell.com.tr/endpoint.openid"/></head></html>' if turkcell_data == http_resp.body: http_resp.body = '<html><head><link rel="openid2.provider" href="https://turkcellid.turkcell.com.tr/endpoint.openid"/></head></html>' claimed_id = http_resp.final_url openid_services = OpenIDServiceEndpoint.fromHTML( claimed_id, http_resp.body) return claimed_id, openid_services
def discoverNoYadis(uri): if re.search("i\.mydocomo\.com", uri): uri = 'http://m.calil.jp/proxy.php?key=sdv62x23&uri='+uri http_resp = fetchers.fetch(uri) if http_resp.status != 200: raise DiscoveryFailure( 'HTTP Response status from identity URL host is not 200. ' 'Got status %r' % (http_resp.status,), http_resp) if re.search("i\.mydocomo\.com", uri): claimed_id = uri else: claimed_id = http_resp.final_url openid_services = OpenIDServiceEndpoint.fromHTML( claimed_id, http_resp.body) return claimed_id, openid_services
def query(self, xri, service_types): """Resolve some services for an XRI. Note: I don't implement any service endpoint selection beyond what the resolver I'm querying does, so the Services I return may well include Services that were not of the types you asked for. May raise fetchers.HTTPFetchingError or L{etxrd.XRDSError} if the fetching or parsing don't go so well. @param xri: An XRI to resolve. @type xri: unicode @param service_types: A list of services types to query for. Service types are URIs. @type service_types: list of str @returns: tuple of (CanonicalID, Service elements) @returntype: (unicode, list of C{ElementTree.Element}s) """ # FIXME: No test coverage! services = [] # Make a seperate request to the proxy resolver for each service # type, as, if it is following Refs, it could return a different # XRDS for each. canonicalID = None for service_type in service_types: url = self.queryURL(xri, service_type) if re.search("i\.mydocomo\.com", url): url = 'http://m.calil.jp/proxy.php?key=sdv62x23&uri='+url response = fetchers.fetch(url) if response.status != 200: # XXX: sucks to fail silently. # print "response not OK:", response continue et = etxrd.parseXRDS(response.body) canonicalID = etxrd.getCanonicalID(xri, et) some_services = list(iterServices(et)) services.extend(some_services) # TODO: # * If we do get hits for multiple service_types, we're almost # certainly going to have duplicated service entries and # broken priority ordering. return canonicalID, services
def query(self, xri, service_types): """Resolve some services for an XRI. Note: I don't implement any service endpoint selection beyond what the resolver I'm querying does, so the Services I return may well include Services that were not of the types you asked for. May raise fetchers.HTTPFetchingError or L{etxrd.XRDSError} if the fetching or parsing don't go so well. @param xri: An XRI to resolve. @type xri: unicode @param service_types: A list of services types to query for. Service types are URIs. @type service_types: list of str @returns: tuple of (CanonicalID, Service elements) @returntype: (unicode, list of C{ElementTree.Element}s) """ # FIXME: No test coverage! services = [] # Make a seperate request to the proxy resolver for each service # type, as, if it is following Refs, it could return a different # XRDS for each. canonicalID = None for service_type in service_types: url = self.queryURL(xri, service_type) response = fetchers.fetch(url) if response.status != 200: # XXX: sucks to fail silently. # print "response not OK:", response continue et = etxrd.parseXRDS(response.body) canonicalID = etxrd.getCanonicalID(xri, et) some_services = list(iterServices(et)) services.extend(some_services) # TODO: # * If we do get hits for multiple service_types, we're almost # certainly going to have duplicated service entries and # broken priority ordering. return canonicalID, services
def discover(uri): """Discover services for a given URI. @param uri: The identity URI as a well-formed http or https URI. The well-formedness and the protocol are not checked, but the results of this function are undefined if those properties do not hold. @return: DiscoveryResult object @raises Exception: Any exception that can be raised by fetching a URL with the given fetcher. """ result = DiscoveryResult(uri) resp = fetchers.fetch(uri, headers={'Accept': YADIS_ACCEPT_HEADER}) if resp.status != 200: raise DiscoveryFailure( 'HTTP Response status from identity URL host is not 200. ' 'Got status %r' % (resp.status,), resp) # Note the URL after following redirects result.normalized_uri = resp.final_url # Attempt to find out where to go to discover the document # or if we already have it result.content_type = resp.headers.get('content-type') # According to the spec, the content-type header must be an exact # match, or else we have to look for an indirection. if (result.content_type and result.content_type.split(';', 1)[0].lower() == YADIS_CONTENT_TYPE): result.xrds_uri = result.normalized_uri else: # Try the header yadis_loc = resp.headers.get(YADIS_HEADER_NAME.lower()) if not yadis_loc: # Parse as HTML if the header is missing. # # XXX: do we want to do something with content-type, like # have a whitelist or a blacklist (for detecting that it's # HTML)? try: yadis_loc = findHTMLMeta(StringIO(resp.body)) except MetaNotFound: pass # At this point, we have not found a YADIS Location URL. We # will return the content that we scanned so that the caller # can try to treat it as an XRDS if it wishes. if yadis_loc: result.xrds_uri = yadis_loc resp = fetchers.fetch(yadis_loc) if resp.status != 200: exc = DiscoveryFailure( 'HTTP Response status from Yadis host is not 200. ' 'Got status %r' % (resp.status,), resp) exc.identity_url = result.normalized_uri raise exc result.content_type = resp.headers.get('content-type') result.response_text = resp.body return result
def test_callFetch(self): """Make sure that fetchers.fetch() uses the default fetcher instance that was set.""" fetchers.setDefaultFetcher(FakeFetcher()) actual = fetchers.fetch('bad://url') self.assertTrue(actual is FakeFetcher.sentinel)
def testMultipleXRD(self): data = fetchers.fetch('http://unittest/test_xrds/multiple-xrd.xml').read() elements = xrds.get_elements(data, []) self.assertEqual(len(elements), 2)
def _get_service(self, url): data = fetchers.fetch(url).read() return xrds.iterServices(xrds.parseXRDS(data))[0]
def test_mismatch(self): data = fetchers.fetch('http://unittest/openid_1_and_2_xrds_bad_delegate.xrds').read() element = xrds.iterServices(xrds.parseXRDS(data))[0] with self.assertRaises(xrds.XRDSError): xrds.getLocalID(element, True, True)
def _getServices(self, types=[], constructor=lambda x: x): data = fetchers.fetch('http://unittest/test_xrds/valid-populated-xrds.xml').read() return [constructor(e) for e in xrds.get_elements(data, types)]
def test_callFetch(self): """Make sure that fetchers.fetch() uses the default fetcher instance that was set.""" fetchers.setDefaultFetcher(FakeFetcher()) actual = fetchers.fetch("bad://url") self.failUnless(actual is FakeFetcher.sentinel)
def testNoXRD(self): data = fetchers.fetch('http://unittest/test_xrds/no-xrd.xml').read() self.assertRaises( xrds.XRDSError, xrds.get_elements, data, [])
def fetch(self, url, body=None, headers=None): return fetchers.fetch(body, headers)