def test_url(self): """ An L{URL} object is flattened to the appropriate representation of itself, whether it is the child of a tag or the value of a tag attribute. """ link = URL.fromString("http://foo/fu?bar=baz&bar=baz#quux%2f") self.assertStringEqual(self.flatten(link), "http://foo/fu?bar=baz&bar=baz#quux%2F") self.assertStringEqual( self.flatten(div[link]), '<div>http://foo/fu?bar=baz&bar=baz#quux%2F</div>') self.assertStringEqual( self.flatten(div(foo=link)), '<div foo="http://foo/fu?bar=baz&bar=baz#quux%2F"></div>') self.assertStringEqual( self.flatten(div[div(foo=link)]), '<div><div foo="http://foo/fu?bar=baz&bar=baz#quux%2F"></div>' '</div>') link = URL.fromString("http://foo/fu?%2f=%7f") self.assertStringEqual(self.flatten(link), "http://foo/fu?%2F=%7F") self.assertStringEqual(self.flatten(div[link]), '<div>http://foo/fu?%2F=%7F</div>') self.assertStringEqual(self.flatten(div(foo=link)), '<div foo="http://foo/fu?%2F=%7F"></div>')
def test_url(self): """ An L{URL} object is flattened to the appropriate representation of itself, whether it is the child of a tag or the value of a tag attribute. """ link = URL.fromString("http://foo/fu?bar=baz&bar=baz#quux%2f") self.assertStringEqual( self.flatten(link), "http://foo/fu?bar=baz&bar=baz#quux%2F") self.assertStringEqual( self.flatten(div[link]), '<div>http://foo/fu?bar=baz&bar=baz#quux%2F</div>') self.assertStringEqual( self.flatten(div(foo=link)), '<div foo="http://foo/fu?bar=baz&bar=baz#quux%2F"></div>') self.assertStringEqual( self.flatten(div[div(foo=link)]), '<div><div foo="http://foo/fu?bar=baz&bar=baz#quux%2F"></div>' '</div>') link = URL.fromString("http://foo/fu?%2f=%7f") self.assertStringEqual( self.flatten(link), "http://foo/fu?%2F=%7F") self.assertStringEqual( self.flatten(div[link]), '<div>http://foo/fu?%2F=%7F</div>') self.assertStringEqual( self.flatten(div(foo=link)), '<div foo="http://foo/fu?%2F=%7F"></div>')
def test_getSelectedTabExactMatch(self): """ Check that L{webnav.getSelectedTab} returns the tab whose C{linkURL} attribute exactly matches the path of the L{nevow.url.URL} it is passed """ tabs = list(webnav.Tab(str(i), None, 0, linkURL="/" + str(i)) for i in xrange(5)) for (i, tab) in enumerate(tabs): selected = webnav.getSelectedTab(tabs, URL.fromString(tab.linkURL)) self.assertIdentical(selected, tab) selected = webnav.getSelectedTab(tabs, URL.fromString("/XYZ")) self.failIf(selected)
def test_getSelectedTabExactMatch(self): """ Check that L{webnav.getSelectedTab} returns the tab whose C{linkURL} attribute exactly matches the path of the L{nevow.url.URL} it is passed """ tabs = list(webnav.Tab(str(i), None, 0, linkURL='/' + str(i)) for i in xrange(5)) for (i, tab) in enumerate(tabs): selected = webnav.getSelectedTab(tabs, URL.fromString(tab.linkURL)) self.assertIdentical(selected, tab) selected = webnav.getSelectedTab(tabs, URL.fromString('/XYZ')) self.failIf(selected)
def test_finish(self): """ L{StylesheetRewritingRequestWrapper.finish} causes all written bytes to be translated with C{_replace} written to the wrapped request. """ stylesheetFormat = """ .foo { background-image: url(%s) } """ originalStylesheet = stylesheetFormat % ("/Foo/bar", ) expectedStylesheet = stylesheetFormat % ("/bar/Foo/bar", ) request = FakeRequest() roots = {request: URL.fromString('/bar/')} wrapper = website.StylesheetRewritingRequestWrapper( request, [], roots.get) wrapper.write(originalStylesheet) wrapper.finish() # Parse and serialize both versions to normalize whitespace so we can # make a comparison. parser = CSSParser() self.assertEqual( parser.parseString(request.accumulator).cssText, parser.parseString(expectedStylesheet).cssText)
def test_finish(self): """ L{StylesheetRewritingRequestWrapper.finish} causes all written bytes to be translated with C{_replace} written to the wrapped request. """ stylesheetFormat = """ .foo { background-image: url(%s) } """ originalStylesheet = stylesheetFormat % ("/Foo/bar",) expectedStylesheet = stylesheetFormat % ("/bar/Foo/bar",) request = FakeRequest() roots = {request: URL.fromString('/bar/')} wrapper = website.StylesheetRewritingRequestWrapper( request, [], roots.get) wrapper.write(originalStylesheet) wrapper.finish() # Parse and serialize both versions to normalize whitespace so we can # make a comparison. parser = CSSParser() self.assertEqual( parser.parseString(request.accumulator).cssText, parser.parseString(expectedStylesheet).cssText)
def __init__(self, url, tries=10, timeout=defaultTimeout, *a, **kw): """ Prepare the download information. Any additional positional or keyword arguments are passed on to C{twisted.web.client.HTTPPageGetter}. @type url: C{nevow.url.URL} or C{unicode} or C{str} @param url: The HTTP URL to attempt to download @type tries: C{int} @param tries: The maximum number of retry attempts before giving up @type timeout: C{float} @param timeout: Timeout value, in seconds, for the page fetch; defaults to L{defaultTimeout} """ if isinstance(url, unicode): url = url.encode('utf-8') if isinstance(url, str): url = URL.fromString(url) self.url = url.anchor(None) self.args = a self.kwargs = kw self.delay = self.initialDelay self.tries = tries self.timeout = timeout
def rendered(request): if request.redirected_to is None: result.callback(request) else: visited.append(request.redirected_to) if visited.index(request.redirected_to) != len(visited) - 1: visited.append(request.redirected_to) result.errback(Exception("Redirect loop: %r" % (visited, ))) elif len(visited) > redirectLimit: result.errback( Exception("Too many redirects: %r" % (visited, ))) else: newHeaders = headers.copy() # Respect redirects location = URL.fromString(request.redirected_to) newHeaders['host'] = location.netloc # Respect cookies cookies.update(request.cookies) # str(URL) shouldn't really do what it does. page = getResource(site, str(location), newHeaders, cookies) page.addCallbacks(rendered, result.errback)
def checkid_setup(registry, requestData, user=None): """ This method will validate and redirect a successful request to its return_to param. If the user isn't logged in, or doesn't have an account, we'll redirect to an internal page. @param registry: the current OpenID registry @type registry: L{OpenIDRegistry} @param requestData: the current request data @type requestData: L{OpenIDRequest} @param user: the current user @type user: L{txopenid.user.User} @return: association response @rtype: L{nevow.url.URL} """ if(user is not None): def _identity_state(): return user.hasIdentity(requestData['openid.identity']) def _trust_state(): return user.trustsRoot(requestData['openid.trust_root']) if not(yield maybeDeferred(_identity_state)): return_to = util.appendQuery(OPENID_IDENTITY_URL, requestData) elif not(yield maybeDeferred(_trust_state)): return_to = util.appendQuery(OPENID_TRUST_URL, requestData) else: return_to = get_login_response(registry, requestData) else: return_to = util.appendQuery(OPENID_LOGIN_URL, requestData) returnValue(URL.fromString(return_to))
def rendered(request): if request.redirected_to is None: result.callback(request) else: visited.append(request.redirected_to) if visited.index(request.redirected_to) != len(visited) - 1: visited.append(request.redirected_to) result.errback(Exception("Redirect loop: %r" % (visited,))) elif len(visited) > redirectLimit: result.errback(Exception("Too many redirects: %r" % (visited,))) else: newHeaders = headers.copy() # Respect redirects location = URL.fromString(request.redirected_to) newHeaders['host'] = location.netloc # Respect cookies cookies.update(request.cookies) # str(URL) shouldn't really do what it does. page = getResource( site, str(location), newHeaders, cookies) page.addCallbacks(rendered, result.errback)
def test_parseURLs(self): """ L{eridanus.iriparse.parseURL} extracts and parses (as a L{nevow.url.URL}) all URIs in a string. """ self.assertEquals( list(iriparse.parseURLs(u'http://google.com/')), [URL.fromString(u'http://google.com/')])
def test_parseURL(self): """ L{eridanus.iriparse.parseURL} extracts and parses (as a L{nevow.url.URL}) the first URI in a string. """ self.assertEquals( iriparse.parseURL( u'http://google.com/ http://foo.bar/ world'), URL.fromString(u'http://google.com/'))
def test_shortURL(self): """ L{StylesheetRewritingRequestWrapper._replace} changes URLs with only one segment so they are beneath the root URL. """ request = object() roots = {request: URL.fromString('/bar/')} wrapper = website.StylesheetRewritingRequestWrapper( request, [], roots.get) self.assertEqual(wrapper._replace('/'), '/bar/')
class Wunderground(object): API_ROOT = URL.fromString('http://api.wunderground.com/auto/wui/geo') @classmethod def current(cls, query): url = cls.API_ROOT.child('WXCurrentObXML').child('index.xml').add( 'query', query) return util.PerseverantDownloader(url).go().addCallback( lambda (data, headers): etree.fromstring(data)).addCallback( WundergroundConditions.fromElement)
def evaluate(self, expn): """ Evaluate an expression. """ url = URL.fromString('http://www.google.com/search?') url = url.add('q', expn + '=') url = url.add('num', '1') d = self._fetch(url) d.addCallback(self._extractResult, expn) return d
def test_replaceMantissa(self): """ L{StylesheetRewritingRequestWrapper._replace} changes URLs of the form I{/Mantissa/foo} to I{<rootURL>/static/mantissa-base/foo}. """ request = object() roots = {request: URL.fromString('/bar/')} wrapper = website.StylesheetRewritingRequestWrapper(request, [], roots.get) self.assertEqual( wrapper._replace('/Mantissa/foo.png'), '/bar/static/mantissa-base/foo.png')
def test_replaceMantissa(self): """ L{StylesheetRewritingRequestWrapper._replace} changes URLs of the form I{/Mantissa/foo} to I{<rootURL>/static/mantissa-base/foo}. """ request = object() roots = {request: URL.fromString('/bar/')} wrapper = website.StylesheetRewritingRequestWrapper( request, [], roots.get) self.assertEqual(wrapper._replace('/Mantissa/foo.png'), '/bar/static/mantissa-base/foo.png')
def rootChild_resetPassword(self, req, webViewer): """ Redirect authenticated users to their settings page (hopefully they have one) when they try to reset their password. This is the wrong way for this functionality to be implemented. See #2524. """ from xmantissa.ixmantissa import IWebTranslator, IPreferenceAggregator return URL.fromString( IWebTranslator(self.store).linkTo( IPreferenceAggregator(self.store).storeID))
def renderHTTP(self, ctx): req = inevow.IRequest(ctx) password = req.args.get('password', [None])[0] if password is None: return Page.renderHTTP(self, ctx) self.original.store.transact(self.original.setPassword, unicode(password)) # XXX TODO: select # proper decoding # strategy. return URL.fromString('/')
def test_replaceOtherOffering(self): """ L{StylesheetRewritingRequestWrapper._replace} changes URLs of the form I{/Something/foo} to I{<rootURL>/static/Something/foo} if C{Something} gives the name of an installed offering with a static content path. """ request = object() roots = {request: URL.fromString('/bar/')} wrapper = website.StylesheetRewritingRequestWrapper( request, ['OfferingName'], roots.get) self.assertEqual(wrapper._replace('/OfferingName/foo.png'), '/bar/static/OfferingName/foo.png')
def test_nonOfferingOnlyGivenPrefix(self): """ L{StylesheetRewritingRequestWrapper._replace} only changes URLs of the form I{/Something/foo} so they are beneath the root URL if C{Something} does not give the name of an installed offering. """ request = object() roots = {request: URL.fromString('/bar/')} wrapper = website.StylesheetRewritingRequestWrapper( request, ['Foo'], roots.get) self.assertEqual(wrapper._replace('/OfferingName/foo.png'), '/bar/OfferingName/foo.png')
def test_shortURL(self): """ L{StylesheetRewritingRequestWrapper._replace} changes URLs with only one segment so they are beneath the root URL. """ request = object() roots = {request: URL.fromString('/bar/')} wrapper = website.StylesheetRewritingRequestWrapper( request, [], roots.get) self.assertEqual( wrapper._replace('/'), '/bar/')
def test_replaceOtherOffering(self): """ L{StylesheetRewritingRequestWrapper._replace} changes URLs of the form I{/Something/foo} to I{<rootURL>/static/Something/foo} if C{Something} gives the name of an installed offering with a static content path. """ request = object() roots = {request: URL.fromString('/bar/')} wrapper = website.StylesheetRewritingRequestWrapper(request, ['OfferingName'], roots.get) self.assertEqual( wrapper._replace('/OfferingName/foo.png'), '/bar/static/OfferingName/foo.png')
def test_nonOfferingOnlyGivenPrefix(self): """ L{StylesheetRewritingRequestWrapper._replace} only changes URLs of the form I{/Something/foo} so they are beneath the root URL if C{Something} does not give the name of an installed offering. """ request = object() roots = {request: URL.fromString('/bar/')} wrapper = website.StylesheetRewritingRequestWrapper( request, ['Foo'], roots.get) self.assertEqual( wrapper._replace('/OfferingName/foo.png'), '/bar/OfferingName/foo.png')
def test_segments(self): """ L{LoginPage.beforeRender} should fill the I{login-action} slot with an L{URL} which includes all the segments given to the L{LoginPage}. """ segments = ('foo', 'bar') page = LoginPage(self.siteStore, segments) page.beforeRender(self.context) loginAction = self.context.locateSlotData('login-action') expectedLocation = URL.fromString('/') for segment in (LOGIN_AVATAR,) + segments: expectedLocation = expectedLocation.child(segment) self.assertEqual(loginAction, expectedLocation)
def test_segments(self): """ L{LoginPage.beforeRender} should fill the I{login-action} slot with an L{URL} which includes all the segments given to the L{LoginPage}. """ segments = ('foo', 'bar') page = LoginPage(self.siteStore, segments) page.beforeRender(self.context) loginAction = self.context.locateSlotData('login-action') expectedLocation = URL.fromString('/') for segment in (LOGIN_AVATAR, ) + segments: expectedLocation = expectedLocation.child(segment) self.assertEqual(loginAction, expectedLocation)
def test_queryArguments(self): """ L{LoginPage.beforeRender} should fill the I{login-action} slot with an L{URL} which includes all the query arguments given to the L{LoginPage}. """ args = {'foo': ['bar']} page = LoginPage(self.siteStore, (), args) page.beforeRender(self.context) loginAction = self.context.locateSlotData('login-action') expectedLocation = URL.fromString('/') expectedLocation = expectedLocation.child(LOGIN_AVATAR) expectedLocation = expectedLocation.add('foo', 'bar') self.assertEqual(loginAction, expectedLocation)
def test_adminRedirect(self): """ When the admin redirect is installed on a store, it should return an URL which should redirect to /private. """ s = Store(self.mktemp()) s.powerUp(self.adminpage.RedirectPlugin(store=s)) m = Mantissa() m.installSite(s, u'localhost', u'', False) root = IMantissaSite(s) viewer = StubViewer() result, segments = root.siteProduceResource(FakeRequest(), tuple(['admin.php']), viewer) self.assertEquals(result, URL.fromString("http://localhost/private"))
def returnedFromProvider(request, sessionDict, here): argsSingle = dict((k, v[0]) for k,v in request.args.items()) c = openid.consumer.consumer.Consumer(sessionDict, store) resp = c.complete(argsSingle, here) if resp.status != 'success': request.setResponseCode(http.UNAUTHORIZED) return "login failed: %s" % resp.message sessionDict['identity'] = resp.identity_url print "returned" pprint.pprint(vars()) redir = dict(URL.fromString(here).queryList()).get('redir') if redir is not None: request.redirect('http://%s%s' % (request.getHeader('host'), redir)) # waiting... else: # todo: clear query params request.redirect(here) return ""
def returnedFromProvider(request, sessionDict, here): argsSingle = dict((k, v[0]) for k, v in request.args.items()) c = openid.consumer.consumer.Consumer(sessionDict, store) resp = c.complete(argsSingle, here) if resp.status != 'success': request.setResponseCode(http.UNAUTHORIZED) return "login failed: %s" % resp.message sessionDict['identity'] = resp.identity_url print "returned" pprint.pprint(vars()) redir = dict(URL.fromString(here).queryList()).get('redir') if redir is not None: request.redirect('http://%s%s' % (request.getHeader('host'), redir)) # waiting... else: # todo: clear query params request.redirect(here) return ""
def getResource(site, uri, headers={}, cookies={}): """ Retrieve the resource at the given URI from C{site}. Return a L{Deferred} which is called back with the request after resource traversal and rendering has finished. @type site: L{NevowSite} @param site: The site object from which to retrieve the resource. @type uri: C{str} @param uri: The absolute path to the resource to retrieve, eg I{/private/12345}. @type headers: C{dict} @param headers: HTTP headers to include in the request. """ headers = headers.copy() cookies = cookies.copy() url = URL.fromString(uri) path = '/' + url.path args = {} for (k, v) in url.queryList(): args.setdefault(k, []).append(v) remainingSegments = tuple(url.pathList()) request = FakeRequest(isSecure=url.scheme == 'https', uri=path, args=args, headers=headers, cookies=cookies, currentSegments=()) requestContext = RequestContext(parent=site.context, tag=request) requestContext.remember((), ICurrentSegments) requestContext.remember(remainingSegments, IRemainingSegments) page = site.getPageContextForRequestContext(requestContext) page.addCallback(renderPage, topLevelContext=lambda tag: tag, reqFactory=lambda: request) page.addCallback(lambda ignored: request) return page
def checkid_immediate(registry, requestData, user=None): """ Validate the provided request. @param registry: the current OpenID registry @type registry: L{OpenIDRegistry} @param requestData: the current request data @type requestData: L{OpenIDRequest} @param user: the current user @type user: L{txopenid.user.User} @return: checkid response @rtype: L{nevow.url.URL} """ if(user is not None): def _identity_state(): return user.hasIdentity(requestData['openid.identity']) def _trust_state(): return user.trustsRoot(requestData['openid.trust_root']) if not(yield maybeDeferred(_identity_state)): return_to = util.appendQuery(requestData['openid.return_to'], { 'openid.mode':'id_res', 'openid.user_setup_url':util.appendQuery(OPENID_IDENTITY_URL, requestData), }) elif not(yield maybeDeferred(_trust_state)): return_to = util.appendQuery(requestData['openid.return_to'], { 'openid.mode':'id_res', 'openid.user_setup_url':util.appendQuery(OPENID_TRUST_URL, requestData), }) else: return_to = get_login_response(registry, requestData) else: return_to = util.appendQuery(requestData['openid.return_to'], { 'openid.mode':'id_res', 'openid.user_setup_url':util.appendQuery(OPENID_LOGIN_URL, requestData), }) returnValue(URL.fromString(return_to))
def getResource(site, uri, headers={}, cookies={}): """ Retrieve the resource at the given URI from C{site}. Return a L{Deferred} which is called back with the request after resource traversal and rendering has finished. @type site: L{NevowSite} @param site: The site object from which to retrieve the resource. @type uri: C{str} @param uri: The absolute path to the resource to retrieve, eg I{/private/12345}. @type headers: C{dict} @param headers: HTTP headers to include in the request. """ headers = headers.copy() cookies = cookies.copy() url = URL.fromString(uri) path = '/' + url.path args = {} for (k, v) in url.queryList(): args.setdefault(k, []).append(v) remainingSegments = tuple(url.pathList()) request = FakeRequest( isSecure=url.scheme == 'https', uri=path, args=args, headers=headers, cookies=cookies, currentSegments=()) requestContext = RequestContext(parent=site.context, tag=request) requestContext.remember((), ICurrentSegments) requestContext.remember(remainingSegments, IRemainingSegments) page = site.getPageContextForRequestContext(requestContext) page.addCallback( renderPage, topLevelContext=lambda tag: tag, reqFactory=lambda: request) page.addCallback(lambda ignored: request) return page
def test_extractStatusIDFromURL(self): """ Status identifiers are extracted from known valid forms of Twitter status URLs, C{None} is extracted for unknown forms. """ expected = [ ('http://twitter.com/bob/status/514431337', '514431337'), ('http://www.twitter.com/bob/status/514431337', '514431337'), ('http://twitter.com/bob/statuses/514431337', '514431337'), ('http://twitter.com/#!/bob/statuses/514431337', '514431337'), ('http://www.twitter.com/#!/bob/statuses/514431337', '514431337'), ('http://www.twitter.com/bob/statuses/hello', None), ('http://www.twitter.com/bob/statuses', None), ('http://somethingnottwitter.com/bob/statuses/514431337', None)] for url, result in expected: assertFn = self.assertEquals if result is None: assertFn = self.assertIdentical assertFn(twitter.extractStatusIDFromURL(URL.fromString(url)), result)
def test_sendEmail(self): """ L{PasswordResetResource.sendEmail} should format a meaningful password reset email. """ resetAddress = '*****@*****.**' resetURI = URL.fromString('http://example.org/resetPassword') userAddress = '*****@*****.**' resetAttempt = self.reset.newAttemptForUser(userAddress.decode('ascii')) _sentEmail = [] self.reset.sendEmail(resetURI, resetAttempt, userAddress, _sendEmail=lambda *args: _sentEmail.append(args)) self.assertEquals(len(_sentEmail), 1) [(sentFrom, sentTo, sentText)] = _sentEmail self.assertEquals(sentFrom, resetAddress) self.assertEquals(sentTo, userAddress) msg = email.message_from_string(sentText) [headerFrom] = msg.get_all('from') [headerTo] = msg.get_all('to') [headerDate] = msg.get_all('date') # Python < 2.5 compatibility try: from email import utils except ImportError: from email import Utils as utils self.assertEquals(utils.parseaddr(headerFrom)[1], resetAddress) self.assertEquals(utils.parseaddr(headerTo)[1], userAddress) self.assertTrue(utils.parsedate_tz(headerDate) is not None, '%r is not a RFC 2822 date' % headerDate) self.assertTrue(not msg.is_multipart()) self.assertIn(flatten(resetURI.child(resetAttempt.key)), msg.get_payload())
from itertools import chain from lxml import etree from nevow.url import URL from eridanus.util import PerseverantDownloader from eridanus.soap import getValueFromXSIType from eridanusstd import errors API_URL = URL.fromString( 'http://duncanmackenzie.net/services/GetXboxInfo.aspx') def getGamertagInfo(gamertag): """ Retrieve information about a gamertag. @type gamertag: C{unicode} @param gamertag: The gamertag to retrieve an overview for @raises errors.InvalidGamertag: If C{gamertag} is invalid or unknown @rtype: C{Deferred} firing with a C{etree.Element} @return: A deferred that fires with the root node for the response document """ def parseResponse((data, headers)): root = etree.fromstring(data) state = root.find('State') if state is None or state.text != 'Valid': raise errors.InvalidGamertag(u'%r is not a valid gamertag' %
import json from twisted.internet import defer from nevow.url import URL from eridanus import util from eridanusstd import errors, defertools from eridanusstd.util import parseHTML HEADERS = { 'Referrer': 'http://trac.slipgate.za.net/Eridanus'} SEARCH_URL = URL.fromString('http://ajax.googleapis.com/ajax/services/search/') class WebSearchQuery(object): """ Encapsulate a Google web search query. Ideally this should be combined with the C{eridanusstd.defertools} module, to lazily pull results from C{self.queue}. With this method, new pages will automatically be fetched when a result set runs dry. @type term: C{unicode} @ivar term: Search term. @type url: C{nevow.url.URL}
def rendered(request): redirectLocation = URL.fromString(request.redirected_to) key, path = redirectLocation.pathList() self.assertTrue(key.startswith(SESSION_KEY)) self.assertEqual(path, "")
from nevow.url import URL from eridanus import util from eridanusstd import errors from eridanusstd.util import parseHTML def handleBadQuoteID(f, quoteID): f.trap(weberror.Error) if int(f.value.status) == http.NOT_FOUND: raise errors.InvalidQuote(quoteID) return f QDB_US_URL = URL.fromString('http://qdb.us/') def qdbUS(quoteID): url = QDB_US_URL.child(quoteID) def extractQuote(tree): quote = tree.find('//form/table/tbody') header = unicode(''.join(quote.find('tr/td').itertext())).strip() text = unicode(''.join(quote.find('tr/td/p').itertext())).strip() yield u'%s -- %s' % (header, url) for line in text.splitlines(): yield line return util.PerseverantDownloader(url).go().addCallback(
from itertools import chain from lxml import etree from nevow.url import URL from eridanus.util import PerseverantDownloader from eridanus.soap import getValueFromXSIType from eridanusstd import errors API_URL = URL.fromString('http://duncanmackenzie.net/services/GetXboxInfo.aspx') def getGamertagInfo(gamertag): """ Retrieve information about a gamertag. @type gamertag: C{unicode} @param gamertag: The gamertag to retrieve an overview for @raises errors.InvalidGamertag: If C{gamertag} is invalid or unknown @rtype: C{Deferred} firing with a C{etree.Element} @return: A deferred that fires with the root node for the response document """ def parseResponse((data, headers)): root = etree.fromstring(data) state = root.find('State') if state is None or state.text != 'Valid': raise errors.InvalidGamertag(u'%r is not a valid gamertag' % (gamertag,))
import csv from decimal import Decimal from StringIO import StringIO from nevow.url import URL from eridanus import util from eridanusstd import errors CURRENCY_URL = URL.fromString('http://download.finance.yahoo.com/d/quotes.csv?f=l1d1t1ba&e=.csv') def currencyExchange(currencyFrom, currencyTo): def gotCSV((data, headers)): row = csv.reader(StringIO(data)).next() lastTradeRate, d, t, bid, ask = row lastTradeRate = Decimal(lastTradeRate) if lastTradeRate == 0: raise errors.InvalidCurrency(u'One of the specified currency codes is invalid') tradeTime = u'%s %s' % (d, t) return Decimal(lastTradeRate), tradeTime url = CURRENCY_URL.add('s', '%s%s=X' % (currencyFrom, currencyTo)) return util.PerseverantDownloader(url).go( ).addCallback(gotCSV)
from nevow.url import URL from eridanus import util from eridanusstd import errors from eridanusstd.util import parseHTML def handleBadQuoteID(f, quoteID): f.trap(weberror.Error) if int(f.value.status) == http.NOT_FOUND: raise errors.InvalidQuote(quoteID) return f QDB_US_URL = URL.fromString('http://qdb.us/') def qdbUS(quoteID): url = QDB_US_URL.child(quoteID) def extractQuote(tree): quote = tree.find('//form/table/tbody') header = unicode(''.join(quote.find('tr/td').itertext())).strip() text = unicode(''.join(quote.find('tr/td/p').itertext())).strip() yield u'%s -- %s' % (header, url) for line in text.splitlines(): yield line return util.PerseverantDownloader(url).go( ).addCallback(lambda (data, headers): parseHTML(data)
""" Utility functions designed for scraping data off IMDB. """ import urllib from twisted.internet import defer from nevow.url import URL from eridanus.util import PerseverantDownloader from eridanusstd.util import parseHTML IMDB_URL = URL.fromString('http://www.imdb.com/') _artifactParams = { 'tvSeries': 'tv', 'tvMovies': 'tvm', 'tvEpisodes': 'ep', 'videos': 'vid', } def searchByTitle(title, exact=True, artifacts=None): """ Search IMDB for artifacts named C{title}. @type title: C{unicode} @param title: The title of the artifact to find @type exact: C{bool}
import lxml.objectify from xml.sax.saxutils import unescape from twisted.internet.defer import succeed from twisted.python.failure import Failure from twisted.web import error as weberror from nevow.url import URL from eridanus import util from eridanusstd import errors, timeutil TWITTER_API = URL.fromString('http://api.twitter.com/1/') TWITTER_SEARCH = URL.fromString('http://search.twitter.com/') def handleError(f): """ Transform a Twitter error message into a C{Failure} wrapping L{eridanusstd.errors.RequestError}. """ f.trap(weberror.Error) try: root = lxml.objectify.fromstring(f.value.response) return Failure( errors.RequestError(root.request, root.error)) except lxml.etree.XMLSyntaxError: return f
def rendered(request): redirectLocation = URL.fromString(request.redirected_to) key, path = redirectLocation.pathList() self.assertTrue(key.startswith(SESSION_KEY)) self.assertEqual(path, '')
""" Utility functions designed for scraping data off IMDB. """ import urllib from twisted.internet import defer from nevow.url import URL from eridanus.util import PerseverantDownloader from eridanusstd.util import parseHTML IMDB_URL = URL.fromString('http://www.imdb.com/') _artifactParams = { 'tvSeries': 'tv', 'tvMovies': 'tvm', 'tvEpisodes': 'ep', 'videos': 'vid', } def searchByTitle(title, exact=True, artifacts=None): """ Search IMDB for artifacts named C{title}. @type title: C{unicode} @param title: The title of the artifact to find
import simplejson as json json # For Pyflakes. except ImportError: import json from twisted.internet import defer from nevow.url import URL from eridanus import util from eridanusstd import errors, defertools from eridanusstd.util import parseHTML HEADERS = {'Referrer': 'http://trac.slipgate.za.net/Eridanus'} SEARCH_URL = URL.fromString('http://ajax.googleapis.com/ajax/services/search/') class WebSearchQuery(object): """ Encapsulate a Google web search query. Ideally this should be combined with the C{eridanusstd.defertools} module, to lazily pull results from C{self.queue}. With this method, new pages will automatically be fetched when a result set runs dry. @type term: C{unicode} @ivar term: Search term. @type url: C{nevow.url.URL} @ivar url: Base search URL.
def assertSelected(tab): selected = webnav.getSelectedTab(tabs, URL.fromString('/a/b/c/d/e')) self.assertIdentical(selected, tab)