示例#1
0
 def test_urllib2 (self):
     self.assertEqual(self.server.request_count, 0)
     try:
         urllib2.urlopen('http://127.0.0.1:%s' % self.port)
         assert False, 'should not get there'
     except urllib2.HTTPError, ex:
         assert ex.code == 501, repr(ex)
示例#2
0
def fetch (url, outq):
    """Fetch a url and push any urls found into a queue."""
    print "fetching", url
    data = ''
    with evy.Timeout(5, False):
        data = urllib2.urlopen(url).read()
    for url_match in url_regex.finditer(data):
        new_url = url_match.group(0)
        outq.put(new_url)
示例#3
0
def fetch(url, outq):
    """Fetch a url and push any urls found into a queue."""
    print "fetching", url
    data = ''
    with evy.Timeout(5, False):
        data = urllib2.urlopen(url).read()
    for url_match in url_regex.finditer(data):
        new_url = url_match.group(0)
        outq.put(new_url)
示例#4
0
def fetch (url, seen, pool):
    """Fetch a url, stick any found urls into the seen set, and
    dispatch any new ones to the pool."""
    print "fetching", url
    data = ''
    with evy.Timeout(5, False):
        data = urllib2.urlopen(url).read()
    for url_match in url_regex.finditer(data):
        new_url = url_match.group(0)
        # only send requests to evy.net so as not to destroy the internet
        if new_url not in seen and 'evy.net' in new_url:
            seen.add(new_url)
            # while this seems stack-recursive, it's actually not:
            # spawned greenthreads start their own stacks
            pool.spawn_n(fetch, new_url, seen, pool)
示例#5
0
def fetch (url):
    print "opening", url
    body = urllib2.urlopen(url).read()
    print "done with", url
    return url, body
示例#6
0
 def raiser ():
     try:
         urllib2.urlopen("http://localhost:%s/echo" % self.port)
     except urllib2.HTTPError, e:
         self.assertEqual(e.code, 400)
         raise
示例#7
0
from evy.patched import urllib2

big_list_of_feeds = """
http://blog.evy.net/feed/
http://rss.slashdot.org/Slashdot/slashdot
http://feeds.boingboing.net/boingboing/iBag
http://feeds.feedburner.com/RockPaperShotgun
http://feeds.penny-arcade.com/pa-mainsite
http://achewood.com/rss.php
http://raysmuckles.blogspot.com/atom.xml
http://rbeef.blogspot.com/atom.xml
http://journeyintoreason.blogspot.com/atom.xml
http://orezscu.blogspot.com/atom.xml
http://feeds2.feedburner.com/AskMetafilter
http://feeds2.feedburner.com/Metafilter
http://stackoverflow.com/feeds
http://feeds.feedburner.com/codinghorror
http://www.tbray.org/ongoing/ongoing.atom
http://www.zeldman.com/feed/
http://ln.hixie.ch/rss/html
"""

url = 'http://localhost:9010/'
result = urllib2.urlopen(url, big_list_of_feeds)
print result.read()