# actually /did/ fire for i in ['Headers', 'Status', 'Error', 'Success']: expect = getattr(self, 'expect%s' % i) if expect: self.assertNotEqual(self.checklist['on%s' % i], 0, '%s => on%s executed 0 times, expected more' % (self.url, i)) elif isinstance(expect, bool): self.assertEqual(self.checklist['on%s' % i], 0, '%s => on%s executed, expected to not' % (self.url, i)) # Now, if an expectDone attribute was provided, then # we should execute it with the request object provided if self.expectDone: self.assertTrue(self.expectDone(self)) UnittestRequest.onDone(self, results, fetcher) if __name__ == '__main__': # This simply serves as an example, and isn't really meant for executing import logging from downpour import logger logger.setLevel(logging.DEBUG) fetcher.push(ExpectRequest('asis/ok.asis', None, { 'content-type': ['text/html'], 'content-length': ['11'] }, ('HTTP/1.1', '200', 'OK'), 'http://localhost:8080/asis/ok.asis', 'Hello world')) fetcher.push(ExpectRequest('asis/301_to_ok.asis', expectURL = [ 'http://localhost:8080/asis/301_to_ok.asis', 'http://localhost:8080/asis/ok.asis' ])) reactor.run()
# see if the last request has finished. with self.pld_lock: self.pldQueue.push_unique( next, time.time() + 20) logger.debug( 'Requests still in flight for %s. Waiting' % next) except Exception: logger.exception('onEmptyQueue failed for %s' % next) continue logger.debug('Returning None (should not happen).') return None if __name__ == '__main__': import logging from downpour import BaseRequest # Turn on logging logger.setLevel(logging.DEBUG) q = qr.Queue('requests') with file('urls.txt') as f: for line in f: q.push(BaseRequest(line.strip())) p = PoliteFetcher(100) p.start()
#! /usr/bin/env python import logging from downpour import logger from downpour.test import run, host from downpour.test import ExpectRequest from downpour.test import ExamineRequest from downpour import BaseFetcher, BaseRequest logger.setLevel(logging.CRITICAL) fetcher = BaseFetcher(stopWhenDone=True) # Try a plain-and-simple request fetcher.push( ExpectRequest('200 Test', host + 'asis/ok.asis', expectHeaders={ 'content-type': ['text/html'], 'content-length': ['11'] }, expectStatus=('HTTP/1.1', '200', 'OK'), expectURL=host + 'asis/ok.asis', expectSuccess='Hello world')) # Try a redirect request, making sure we get # every url we expect to get. fetcher.push( ExpectRequest( '301 Redirect Test', host + 'asis/301_to_ok.asis',
#! /usr/bin/env python import logging from downpour import logger from downpour.test import run, host from downpour.test import ExpectRequest from downpour.test import ExamineRequest from downpour import BaseFetcher, BaseRequest logger.setLevel(logging.CRITICAL) fetcher = BaseFetcher(stopWhenDone=True) # Try a plain-and-simple request fetcher.push(ExpectRequest('200 Test', host + 'asis/ok.asis', expectHeaders = { 'content-type': ['text/html'], 'content-length': ['11'] }, expectStatus = ('HTTP/1.1', '200', 'OK'), expectURL = host + 'asis/ok.asis', expectSuccess = 'Hello world')) # Try a redirect request, making sure we get # every url we expect to get. fetcher.push(ExpectRequest('301 Redirect Test', host + 'asis/301_to_ok.asis', expectURL = [ host + 'asis/301_to_ok.asis', host + 'asis/ok.asis' ])) fetcher.push(ExpectRequest('302 Redirect Test', host + 'asis/302_to_ok.asis', expectURL = [ host + 'asis/302_to_ok.asis',