def main(): logging.basicConfig(level=logging.DEBUG) parser = OptionParser() parser.add_option('-t', '--test', help='Run only specified tests') parser.add_option('--transport', help='Test specified transport', default='grab.transport.curl.CurlTransport') parser.add_option('--extra', action='store_true', default=False, help='Run extra tests for specific backends') parser.add_option('--test-grab', action='store_true', default=False, help='Run tests for Grab::Spider') parser.add_option('--test-spider', action='store_true', default=False, help='Run tests for Grab') parser.add_option('--test-all', action='store_true', default=False, help='Run tests for both Grab and Grab::Spider') opts, args = parser.parse_args() test.util.GRAB_TRANSPORT = opts.transport prepare_test_environment() test_list = [] if opts.test_all: test_list += GRAB_TEST_LIST test_list += SPIDER_TEST_LIST if opts.extra: test_list += GRAB_EXTRA_TEST_LIST test_list += SPIDER_EXTRA_TEST_LIST if opts.test_grab: test_list += GRAB_TEST_LIST if opts.extra: test_list += GRAB_EXTRA_TEST_LIST if opts.test_spider: test_list += SPIDER_TEST_LIST if opts.extra: test_list += SPIDER_EXTRA_TEST_LIST if opts.test: test_list += [opts.test] # Check tests integrity # Ensure that all test modules are imported correctly for path in test_list: __import__(path, None, None, ['foo']) loader = unittest.TestLoader() suite = loader.loadTestsFromNames(test_list) runner = unittest.TextTestRunner() start_server() result = runner.run(suite) stop_server() clear_test_environment() if result.wasSuccessful(): sys.exit(0) else: sys.exit(1)
def main(): logging.basicConfig(level=logging.DEBUG) parser = OptionParser() parser.add_option('-t', '--test', help='Run only specified tests') parser.add_option('--test-grab', action='store_true', default=False, help='Run tests for Grab::Spider') parser.add_option('--test-spider', action='store_true', default=False, help='Run tests for Grab') parser.add_option('--test-all', action='store_true', default=False, help='Run tests for both Grab and Grab::Spider') parser.add_option('--backend-mongo', action='store_true', default=False, help='Run extra tests that depends on mongodb') parser.add_option('--backend-redis', action='store_true', default=False, help='Run extra tests that depends on redis') parser.add_option('--backend-mysql', action='store_true', default=False, help='Run extra tests that depends on mysql') parser.add_option('--backend-postgresql', action='store_true', default=False, help='Run extra tests that depends on postgresql') opts, args = parser.parse_args() if opts.backend_mongo: GLOBAL['backends'].append('mongo') if opts.backend_redis: GLOBAL['backends'].append('redis') if opts.backend_mysql: GLOBAL['backends'].append('mysql') if opts.backend_postgresql: GLOBAL['backends'].append('postgresql') prepare_test_environment() test_list = [] if opts.test_all: test_list += GRAB_TEST_LIST test_list += SPIDER_TEST_LIST if opts.test_grab: test_list += GRAB_TEST_LIST if opts.test_spider: test_list += SPIDER_TEST_LIST if opts.test: test_list += [opts.test] # Check tests integrity # Ensure that all test modules are imported correctly for path in test_list: __import__(path, None, None, ['foo']) loader = unittest.TestLoader() suite = unittest.TestSuite() for path in test_list: mod_suite = loader.loadTestsFromName(path) for some_suite in mod_suite: for test in some_suite: if (not hasattr(test, '_backend') or test._backend in GLOBAL['backends']): suite.addTest(test) runner = unittest.TextTestRunner() result = runner.run(suite) clear_test_environment() if result.wasSuccessful(): sys.exit(0) else: sys.exit(1)
def main(): logging.basicConfig(level=logging.DEBUG) parser = OptionParser() parser.add_option('-t', '--test', help='Run only specified tests') parser.add_option('--test-grab', action='store_true', default=False, help='Run tests for Grab::Spider') parser.add_option('--test-spider', action='store_true', default=False, help='Run tests for Grab') parser.add_option('--test-all', action='store_true', default=False, help='Run tests for both Grab and Grab::Spider') parser.add_option('--backend-mongo', action='store_true', default=False, help='Run extra tests that depends on mongodb') parser.add_option('--backend-redis', action='store_true', default=False, help='Run extra tests that depends on redis') parser.add_option('--backend-mysql', action='store_true', default=False, help='Run extra tests that depends on mysql') parser.add_option('--backend-postgresql', action='store_true', default=False, help='Run extra tests that depends on postgresql') parser.add_option('--mp-mode', action='store_true', default=False, help='Enable multiprocess mode in spider tests') parser.add_option('--profile', action='store_true', default=False, help='Do profiling') opts, args = parser.parse_args() if opts.backend_mongo: GLOBAL['backends'].append('mongo') if opts.backend_redis: GLOBAL['backends'].append('redis') if opts.backend_mysql: GLOBAL['backends'].append('mysql') if opts.backend_postgresql: GLOBAL['backends'].append('postgresql') prepare_test_environment() test_list = [] if opts.test_all: test_list += GRAB_TEST_LIST test_list += SPIDER_TEST_LIST if opts.test_grab: test_list += GRAB_TEST_LIST if opts.test_spider: test_list += SPIDER_TEST_LIST if opts.test: test_list += [opts.test] GLOBAL['mp_mode'] = opts.mp_mode # Check tests integrity # Ensure that all test modules are imported correctly for path in test_list: __import__(path, None, None, ['foo']) loader = unittest.TestLoader() suite = unittest.TestSuite() for path in test_list: mod_suite = loader.loadTestsFromName(path) for some_suite in mod_suite: for test in some_suite: if (not hasattr(test, '_backend') or test._backend in GLOBAL['backends']): suite.addTest(test) runner = unittest.TextTestRunner() #start_server() if opts.profile: import cProfile import pyprof2calltree import pstats profile_tree_file = 'var/test.prof.out' prof = cProfile.Profile() result = prof.runcall(runner.run, suite) stats = pstats.Stats(prof) stats.strip_dirs() pyprof2calltree.convert(stats, profile_tree_file) else: result = runner.run(suite) clear_test_environment() if result.wasSuccessful(): sys.exit(0) else: sys.exit(1)
def main(): logging.basicConfig(level=logging.DEBUG) parser = OptionParser() parser.add_option('-t', '--test', help='Run only specified tests') parser.add_option('--transport', help='Test specified transport', default='grab.transport.curl.CurlTransport') parser.add_option('--extra', action='store_true', default=False, help='Run extra tests for specific backends') parser.add_option('--test-grab', action='store_true', default=False, help='Run tests for Grab::Spider') parser.add_option('--test-spider', action='store_true', default=False, help='Run tests for Grab') parser.add_option('--test-all', action='store_true', default=False, help='Run tests for both Grab and Grab::Spider') parser.add_option('--test-kit', action='store_true', default=False, help='Run tests for Grab with WebKit transport') opts, args = parser.parse_args() test.util.GRAB_TRANSPORT = opts.transport if opts.test_kit: test.util.GRAB_TRANSPORT = 'grab.transport.kit.KitTransport' prepare_test_environment() test_list = [] if opts.test_all: test_list += GRAB_TEST_LIST test_list += SPIDER_TEST_LIST if opts.extra: test_list += GRAB_EXTRA_TEST_LIST test_list += SPIDER_EXTRA_TEST_LIST if opts.test_grab: test_list += GRAB_TEST_LIST if opts.extra: test_list += GRAB_EXTRA_TEST_LIST if opts.test_kit: test_list += KIT_TEST_LIST if opts.extra: test_list += KIT_EXTRA_TEST_LIST if opts.test_spider: test_list += SPIDER_TEST_LIST if opts.extra: test_list += SPIDER_EXTRA_TEST_LIST if opts.test: test_list += [opts.test] # Check tests integrity # Ensure that all test modules are imported correctly for path in test_list: __import__(path, None, None, ['foo']) loader = unittest.TestLoader() suite = loader.loadTestsFromNames(test_list) runner = unittest.TextTestRunner() start_server() result = runner.run(suite) stop_server() clear_test_environment() if result.wasSuccessful(): sys.exit(0) else: sys.exit(1)
def main(): logging.basicConfig(level=logging.DEBUG) parser = OptionParser() parser.add_option('-t', '--test', help='Run only specified tests') parser.add_option('--transport', default='pycurl') parser.add_option('--test-grab', action='store_true', default=False, help='Run tests for Grab::Spider') parser.add_option('--test-spider', action='store_true', default=False, help='Run tests for Grab') parser.add_option('--test-all', action='store_true', default=False, help='Run tests for both Grab and Grab::Spider') parser.add_option('--backend-mongo', action='store_true', default=False, help='Run extra tests that depends on mongodb') parser.add_option('--backend-redis', action='store_true', default=False, help='Run extra tests that depends on redis') parser.add_option('--backend-mysql', action='store_true', default=False, help='Run extra tests that depends on mysql') parser.add_option('--backend-postgresql', action='store_true', default=False, help='Run extra tests that depends on postgresql') parser.add_option('--mp-mode', action='store_true', default=False, help='Enable multiprocess mode in spider tests') parser.add_option('--profile', action='store_true', default=False, help='Do profiling') opts, args = parser.parse_args() GLOBAL['transport'] = opts.transport if opts.backend_mongo: GLOBAL['backends'].append('mongo') if opts.backend_redis: GLOBAL['backends'].append('redis') if opts.backend_mysql: GLOBAL['backends'].append('mysql') if opts.backend_postgresql: GLOBAL['backends'].append('postgresql') prepare_test_environment() test_list = [] if opts.test_all: test_list += GRAB_TEST_LIST test_list += SPIDER_TEST_LIST if opts.test_grab: test_list += GRAB_TEST_LIST if opts.test_spider: test_list += SPIDER_TEST_LIST if opts.test: test_list += [opts.test] GLOBAL['mp_mode'] = opts.mp_mode # Check tests integrity # Ensure that all test modules are imported correctly for path in test_list: __import__(path, None, None, ['foo']) loader = unittest.TestLoader() suite = unittest.TestSuite() for path in test_list: mod_suite = loader.loadTestsFromName(path) for some_suite in mod_suite: for test in some_suite: if (not hasattr(test, '_backend') or test._backend in GLOBAL['backends']): suite.addTest(test) runner = unittest.TextTestRunner() #start_server() if opts.profile: import cProfile import pyprof2calltree import pstats profile_tree_file = 'var/test.prof.out' prof = cProfile.Profile() result = prof.runcall(runner.run, suite) stats = pstats.Stats(prof) stats.strip_dirs() pyprof2calltree.convert(stats, profile_tree_file) else: result = runner.run(suite) clear_test_environment() if result.wasSuccessful(): sys.exit(0) else: sys.exit(1)
def main(): logging.basicConfig(level=logging.DEBUG) parser = OptionParser() parser.add_option("-t", "--test", help="Run only specified tests") parser.add_option("--test-grab", action="store_true", default=False, help="Run tests for Grab::Spider") parser.add_option("--test-spider", action="store_true", default=False, help="Run tests for Grab") parser.add_option("--test-all", action="store_true", default=False, help="Run tests for both Grab and Grab::Spider") parser.add_option( "--backend-mongo", action="store_true", default=False, help="Run extra tests that depends on mongodb" ) parser.add_option( "--backend-redis", action="store_true", default=False, help="Run extra tests that depends on redis" ) parser.add_option( "--backend-mysql", action="store_true", default=False, help="Run extra tests that depends on mysql" ) parser.add_option( "--backend-postgresql", action="store_true", default=False, help="Run extra tests that depends on postgresql" ) parser.add_option("--mp-mode", action="store_true", default=False, help="Enable multiprocess mode in spider tests") opts, args = parser.parse_args() if opts.backend_mongo: GLOBAL["backends"].append("mongo") if opts.backend_redis: GLOBAL["backends"].append("redis") if opts.backend_mysql: GLOBAL["backends"].append("mysql") if opts.backend_postgresql: GLOBAL["backends"].append("postgresql") prepare_test_environment() test_list = [] if opts.test_all: test_list += GRAB_TEST_LIST test_list += SPIDER_TEST_LIST if opts.test_grab: test_list += GRAB_TEST_LIST if opts.test_spider: test_list += SPIDER_TEST_LIST if opts.test: test_list += [opts.test] GLOBAL["mp_mode"] = opts.mp_mode # Check tests integrity # Ensure that all test modules are imported correctly for path in test_list: __import__(path, None, None, ["foo"]) loader = unittest.TestLoader() suite = unittest.TestSuite() for path in test_list: mod_suite = loader.loadTestsFromName(path) for some_suite in mod_suite: for test in some_suite: if not hasattr(test, "_backend") or test._backend in GLOBAL["backends"]: suite.addTest(test) runner = unittest.TextTestRunner() # start_server() result = runner.run(suite) # stop_server() clear_test_environment() if result.wasSuccessful(): sys.exit(0) else: sys.exit(1)
def main(): logging.basicConfig(level=logging.DEBUG) parser = OptionParser() parser.add_option("-t", "--test", help="Run only specified tests") parser.add_option("--transport", help="Test specified transport", default="grab.transport.curl.CurlTransport") parser.add_option("--extra", action="store_true", default=False, help="Run extra tests for specific backends") parser.add_option("--test-grab", action="store_true", default=False, help="Run tests for Grab::Spider") parser.add_option("--test-spider", action="store_true", default=False, help="Run tests for Grab") parser.add_option("--test-all", action="store_true", default=False, help="Run tests for both Grab and Grab::Spider") parser.add_option("--test-kit", action="store_true", default=False, help="Run tests for Grab with WebKit transport") opts, args = parser.parse_args() test.util.GRAB_TRANSPORT = opts.transport if opts.test_kit: test.util.GRAB_TRANSPORT = "grab.transport.kit.KitTransport" prepare_test_environment() test_list = [] if opts.test_all: test_list += GRAB_TEST_LIST test_list += SPIDER_TEST_LIST if opts.extra: test_list += GRAB_EXTRA_TEST_LIST test_list += SPIDER_EXTRA_TEST_LIST if opts.test_grab: test_list += GRAB_TEST_LIST if opts.extra: test_list += GRAB_EXTRA_TEST_LIST if opts.test_kit: test_list += KIT_TEST_LIST if opts.extra: test_list += KIT_EXTRA_TEST_LIST if opts.test_spider: test_list += SPIDER_TEST_LIST if opts.extra: test_list += SPIDER_EXTRA_TEST_LIST if opts.test: test_list += [opts.test] # Check tests integrity # Ensure that all test modules are imported correctly for path in test_list: __import__(path, None, None, ["foo"]) loader = unittest.TestLoader() suite = loader.loadTestsFromNames(test_list) runner = unittest.TextTestRunner() start_server() result = runner.run(suite) stop_server() clear_test_environment() if result.wasSuccessful(): sys.exit(0) else: sys.exit(1)