def test_urllib2 (self): self.assertEqual(self.server.request_count, 0) try: urllib2.urlopen('http://127.0.0.1:%s' % self.port) assert False, 'should not get there' except urllib2.HTTPError, ex: assert ex.code == 501, repr(ex)
def fetch (url, outq): """Fetch a url and push any urls found into a queue.""" print "fetching", url data = '' with evy.Timeout(5, False): data = urllib2.urlopen(url).read() for url_match in url_regex.finditer(data): new_url = url_match.group(0) outq.put(new_url)
def fetch(url, outq): """Fetch a url and push any urls found into a queue.""" print "fetching", url data = '' with evy.Timeout(5, False): data = urllib2.urlopen(url).read() for url_match in url_regex.finditer(data): new_url = url_match.group(0) outq.put(new_url)
def fetch (url, seen, pool): """Fetch a url, stick any found urls into the seen set, and dispatch any new ones to the pool.""" print "fetching", url data = '' with evy.Timeout(5, False): data = urllib2.urlopen(url).read() for url_match in url_regex.finditer(data): new_url = url_match.group(0) # only send requests to evy.net so as not to destroy the internet if new_url not in seen and 'evy.net' in new_url: seen.add(new_url) # while this seems stack-recursive, it's actually not: # spawned greenthreads start their own stacks pool.spawn_n(fetch, new_url, seen, pool)
def fetch (url): print "opening", url body = urllib2.urlopen(url).read() print "done with", url return url, body
def raiser (): try: urllib2.urlopen("http://localhost:%s/echo" % self.port) except urllib2.HTTPError, e: self.assertEqual(e.code, 400) raise
from evy.patched import urllib2 big_list_of_feeds = """ http://blog.evy.net/feed/ http://rss.slashdot.org/Slashdot/slashdot http://feeds.boingboing.net/boingboing/iBag http://feeds.feedburner.com/RockPaperShotgun http://feeds.penny-arcade.com/pa-mainsite http://achewood.com/rss.php http://raysmuckles.blogspot.com/atom.xml http://rbeef.blogspot.com/atom.xml http://journeyintoreason.blogspot.com/atom.xml http://orezscu.blogspot.com/atom.xml http://feeds2.feedburner.com/AskMetafilter http://feeds2.feedburner.com/Metafilter http://stackoverflow.com/feeds http://feeds.feedburner.com/codinghorror http://www.tbray.org/ongoing/ongoing.atom http://www.zeldman.com/feed/ http://ln.hixie.ch/rss/html """ url = 'http://localhost:9010/' result = urllib2.urlopen(url, big_list_of_feeds) print result.read()