def test_urllib2 (self):
self.assertEqual(self.server.request_count, 0)
try:
urllib2.urlopen('http://127.0.0.1:%s' % self.port)
assert False, 'should not get there'
except urllib2.HTTPError, ex:
assert ex.code == 501, repr(ex)
def fetch (url, outq):
"""Fetch a url and push any urls found into a queue."""
print "fetching", url
data = ''
with evy.Timeout(5, False):
data = urllib2.urlopen(url).read()
for url_match in url_regex.finditer(data):
new_url = url_match.group(0)
outq.put(new_url)
def fetch(url, outq):
"""Fetch a url and push any urls found into a queue."""
print "fetching", url
data = ''
with evy.Timeout(5, False):
data = urllib2.urlopen(url).read()
for url_match in url_regex.finditer(data):
new_url = url_match.group(0)
outq.put(new_url)
def fetch (url, seen, pool):
"""Fetch a url, stick any found urls into the seen set, and
dispatch any new ones to the pool."""
print "fetching", url
data = ''
with evy.Timeout(5, False):
data = urllib2.urlopen(url).read()
for url_match in url_regex.finditer(data):
new_url = url_match.group(0)
# only send requests to evy.net so as not to destroy the internet
if new_url not in seen and 'evy.net' in new_url:
seen.add(new_url)
# while this seems stack-recursive, it's actually not:
# spawned greenthreads start their own stacks
pool.spawn_n(fetch, new_url, seen, pool)
def fetch (url):
print "opening", url
body = urllib2.urlopen(url).read()
print "done with", url
return url, body
def raiser ():
try:
urllib2.urlopen("http://localhost:%s/echo" % self.port)
except urllib2.HTTPError, e:
self.assertEqual(e.code, 400)
raise
from evy.patched import urllib2 big_list_of_feeds = """ http://blog.evy.net/feed/ http://rss.slashdot.org/Slashdot/slashdot http://feeds.boingboing.net/boingboing/iBag http://feeds.feedburner.com/RockPaperShotgun http://feeds.penny-arcade.com/pa-mainsite http://achewood.com/rss.php http://raysmuckles.blogspot.com/atom.xml http://rbeef.blogspot.com/atom.xml http://journeyintoreason.blogspot.com/atom.xml http://orezscu.blogspot.com/atom.xml http://feeds2.feedburner.com/AskMetafilter http://feeds2.feedburner.com/Metafilter http://stackoverflow.com/feeds http://feeds.feedburner.com/codinghorror http://www.tbray.org/ongoing/ongoing.atom http://www.zeldman.com/feed/ http://ln.hixie.ch/rss/html """ url = 'http://localhost:9010/' result = urllib2.urlopen(url, big_list_of_feeds) print result.read()
