def test_generator_with_invalid_url(self): class SomeSpider(Spider): def task_generator(self): yield Task('page', url=INVALID_URL) from grab.spider.base import logger_verbose logger_verbose.setLevel(logging.DEBUG) bot = SomeSpider() bot.run()
def main(): setup_logging() parser = OptionParser() parser.add_option('-t', '--test', help='Run only specified tests') parser.add_option('--grab-transport', default='pycurl') parser.add_option('--network-service', default='threaded') parser.add_option('--test-grab', action='store_true', default=False, help='Run tests for Grab::Spider') parser.add_option('--test-spider', action='store_true', default=False, help='Run tests for Grab') parser.add_option('--test-all', action='store_true', default=False, help='Run tests for both Grab and Grab::Spider') parser.add_option('--backend-mongodb', action='store_true', default=False, help='Run extra tests that depends on mongodb') parser.add_option('--backend-redis', action='store_true', default=False, help='Run extra tests that depends on redis') parser.add_option('--backend-mysql', action='store_true', default=False, help='Run extra tests that depends on mysql') parser.add_option('--backend-postgresql', action='store_true', default=False, help='Run extra tests that depends on postgresql') parser.add_option('--mp-mode', action='store_true', default=False, help='Enable multiprocess mode in spider tests') parser.add_option('--profile', action='store_true', default=False, help='Do profiling') parser.add_option('-v', '--verbose', action='store_true', default=False, help='Enable verbose logging') opts, _ = parser.parse_args() GLOBAL['grab_transport'] = opts.grab_transport GLOBAL['network_service'] = opts.network_service if opts.backend_mongodb: GLOBAL['backends'].append('mongodb') if opts.backend_redis: GLOBAL['backends'].append('redis') if opts.backend_mysql: GLOBAL['backends'].append('mysql') if opts.backend_postgresql: GLOBAL['backends'].append('postgresql') test_list = [] if opts.test_all: test_list += GRAB_TEST_LIST test_list += SPIDER_TEST_LIST if opts.test_grab: test_list += GRAB_TEST_LIST if opts.test_spider: test_list += SPIDER_TEST_LIST if opts.test: test_list += [opts.test] if opts.verbose: from grab.spider.base import logger_verbose logger_verbose.setLevel(logging.DEBUG) GLOBAL['mp_mode'] = opts.mp_mode # Check tests integrity # Ensure that all test modules are imported correctly for path in test_list: __import__(path, None, None, ['foo']) loader = unittest.TestLoader() suite = unittest.TestSuite() for path in test_list: mod_suite = loader.loadTestsFromName(path) for some_suite in mod_suite: for test in some_suite: if (not hasattr(test, 'backend') or test.backend in GLOBAL['backends']): suite.addTest(test) runner = unittest.TextTestRunner() if opts.profile: import cProfile import pyprof2calltree import pstats profile_tree_file = 'var/test.prof.out' prof = cProfile.Profile() result = prof.runcall(runner.run, suite) stats = pstats.Stats(prof) stats.strip_dirs() pyprof2calltree.convert(stats, profile_tree_file) else: result = runner.run(suite) th_list = list(threading.enumerate()) print('Active threads (%d):' % len(th_list)) for th in th_list: print('Thread: %s (isAlive:%s)' % (th, th.isAlive())) if result.wasSuccessful(): sys.exit(0) else: sys.exit(1)