def main():
logging.basicConfig(level=logging.DEBUG)
parser = OptionParser()
parser.add_option('-t', '--test', help='Run only specified tests')
parser.add_option('--transport', help='Test specified transport',
default='grab.transport.curl.CurlTransport')
parser.add_option('--extra', action='store_true',
default=False, help='Run extra tests for specific backends')
parser.add_option('--test-grab', action='store_true',
default=False, help='Run tests for Grab::Spider')
parser.add_option('--test-spider', action='store_true',
default=False, help='Run tests for Grab')
parser.add_option('--test-all', action='store_true',
default=False, help='Run tests for both Grab and Grab::Spider')
opts, args = parser.parse_args()
test.util.GRAB_TRANSPORT = opts.transport
prepare_test_environment()
test_list = []
if opts.test_all:
test_list += GRAB_TEST_LIST
test_list += SPIDER_TEST_LIST
if opts.extra:
test_list += GRAB_EXTRA_TEST_LIST
test_list += SPIDER_EXTRA_TEST_LIST
if opts.test_grab:
test_list += GRAB_TEST_LIST
if opts.extra:
test_list += GRAB_EXTRA_TEST_LIST
if opts.test_spider:
test_list += SPIDER_TEST_LIST
if opts.extra:
test_list += SPIDER_EXTRA_TEST_LIST
if opts.test:
test_list += [opts.test]
# Check tests integrity
# Ensure that all test modules are imported correctly
for path in test_list:
__import__(path, None, None, ['foo'])
loader = unittest.TestLoader()
suite = loader.loadTestsFromNames(test_list)
runner = unittest.TextTestRunner()
start_server()
result = runner.run(suite)
stop_server()
clear_test_environment()
if result.wasSuccessful():
sys.exit(0)
else:
sys.exit(1)
def main():
logging.basicConfig(level=logging.DEBUG)
parser = OptionParser()
parser.add_option('-t', '--test', help='Run only specified tests')
parser.add_option('--test-grab', action='store_true',
default=False, help='Run tests for Grab::Spider')
parser.add_option('--test-spider', action='store_true',
default=False, help='Run tests for Grab')
parser.add_option('--test-all', action='store_true',
default=False, help='Run tests for both Grab and Grab::Spider')
parser.add_option('--backend-mongo', action='store_true',
default=False, help='Run extra tests that depends on mongodb')
parser.add_option('--backend-redis', action='store_true',
default=False, help='Run extra tests that depends on redis')
parser.add_option('--backend-mysql', action='store_true',
default=False, help='Run extra tests that depends on mysql')
parser.add_option('--backend-postgresql', action='store_true',
default=False, help='Run extra tests that depends on postgresql')
opts, args = parser.parse_args()
if opts.backend_mongo:
GLOBAL['backends'].append('mongo')
if opts.backend_redis:
GLOBAL['backends'].append('redis')
if opts.backend_mysql:
GLOBAL['backends'].append('mysql')
if opts.backend_postgresql:
GLOBAL['backends'].append('postgresql')
prepare_test_environment()
test_list = []
if opts.test_all:
test_list += GRAB_TEST_LIST
test_list += SPIDER_TEST_LIST
if opts.test_grab:
test_list += GRAB_TEST_LIST
if opts.test_spider:
test_list += SPIDER_TEST_LIST
if opts.test:
test_list += [opts.test]
# Check tests integrity
# Ensure that all test modules are imported correctly
for path in test_list:
__import__(path, None, None, ['foo'])
loader = unittest.TestLoader()
suite = unittest.TestSuite()
for path in test_list:
mod_suite = loader.loadTestsFromName(path)
for some_suite in mod_suite:
for test in some_suite:
if (not hasattr(test, '_backend') or
test._backend in GLOBAL['backends']):
suite.addTest(test)
runner = unittest.TextTestRunner()
result = runner.run(suite)
clear_test_environment()
if result.wasSuccessful():
sys.exit(0)
else:
sys.exit(1)
def main():
logging.basicConfig(level=logging.DEBUG)
parser = OptionParser()
parser.add_option('-t', '--test', help='Run only specified tests')
parser.add_option('--test-grab', action='store_true',
default=False, help='Run tests for Grab::Spider')
parser.add_option('--test-spider', action='store_true',
default=False, help='Run tests for Grab')
parser.add_option('--test-all', action='store_true',
default=False,
help='Run tests for both Grab and Grab::Spider')
parser.add_option('--backend-mongo', action='store_true',
default=False,
help='Run extra tests that depends on mongodb')
parser.add_option('--backend-redis', action='store_true',
default=False,
help='Run extra tests that depends on redis')
parser.add_option('--backend-mysql', action='store_true',
default=False,
help='Run extra tests that depends on mysql')
parser.add_option('--backend-postgresql', action='store_true',
default=False,
help='Run extra tests that depends on postgresql')
parser.add_option('--mp-mode', action='store_true', default=False,
help='Enable multiprocess mode in spider tests')
parser.add_option('--profile', action='store_true', default=False,
help='Do profiling')
opts, args = parser.parse_args()
if opts.backend_mongo:
GLOBAL['backends'].append('mongo')
if opts.backend_redis:
GLOBAL['backends'].append('redis')
if opts.backend_mysql:
GLOBAL['backends'].append('mysql')
if opts.backend_postgresql:
GLOBAL['backends'].append('postgresql')
prepare_test_environment()
test_list = []
if opts.test_all:
test_list += GRAB_TEST_LIST
test_list += SPIDER_TEST_LIST
if opts.test_grab:
test_list += GRAB_TEST_LIST
if opts.test_spider:
test_list += SPIDER_TEST_LIST
if opts.test:
test_list += [opts.test]
GLOBAL['mp_mode'] = opts.mp_mode
# Check tests integrity
# Ensure that all test modules are imported correctly
for path in test_list:
__import__(path, None, None, ['foo'])
loader = unittest.TestLoader()
suite = unittest.TestSuite()
for path in test_list:
mod_suite = loader.loadTestsFromName(path)
for some_suite in mod_suite:
for test in some_suite:
if (not hasattr(test, '_backend') or
test._backend in GLOBAL['backends']):
suite.addTest(test)
runner = unittest.TextTestRunner()
#start_server()
if opts.profile:
import cProfile
import pyprof2calltree
import pstats
profile_tree_file = 'var/test.prof.out'
prof = cProfile.Profile()
result = prof.runcall(runner.run, suite)
stats = pstats.Stats(prof)
stats.strip_dirs()
pyprof2calltree.convert(stats, profile_tree_file)
else:
result = runner.run(suite)
clear_test_environment()
if result.wasSuccessful():
sys.exit(0)
else:
sys.exit(1)
def main():
logging.basicConfig(level=logging.DEBUG)
parser = OptionParser()
parser.add_option('-t', '--test', help='Run only specified tests')
parser.add_option('--transport', help='Test specified transport',
default='grab.transport.curl.CurlTransport')
parser.add_option('--extra', action='store_true',
default=False, help='Run extra tests for specific backends')
parser.add_option('--test-grab', action='store_true',
default=False, help='Run tests for Grab::Spider')
parser.add_option('--test-spider', action='store_true',
default=False, help='Run tests for Grab')
parser.add_option('--test-all', action='store_true',
default=False, help='Run tests for both Grab and Grab::Spider')
parser.add_option('--test-kit', action='store_true',
default=False, help='Run tests for Grab with WebKit transport')
opts, args = parser.parse_args()
test.util.GRAB_TRANSPORT = opts.transport
if opts.test_kit:
test.util.GRAB_TRANSPORT = 'grab.transport.kit.KitTransport'
prepare_test_environment()
test_list = []
if opts.test_all:
test_list += GRAB_TEST_LIST
test_list += SPIDER_TEST_LIST
if opts.extra:
test_list += GRAB_EXTRA_TEST_LIST
test_list += SPIDER_EXTRA_TEST_LIST
if opts.test_grab:
test_list += GRAB_TEST_LIST
if opts.extra:
test_list += GRAB_EXTRA_TEST_LIST
if opts.test_kit:
test_list += KIT_TEST_LIST
if opts.extra:
test_list += KIT_EXTRA_TEST_LIST
if opts.test_spider:
test_list += SPIDER_TEST_LIST
if opts.extra:
test_list += SPIDER_EXTRA_TEST_LIST
if opts.test:
test_list += [opts.test]
# Check tests integrity
# Ensure that all test modules are imported correctly
for path in test_list:
__import__(path, None, None, ['foo'])
loader = unittest.TestLoader()
suite = loader.loadTestsFromNames(test_list)
runner = unittest.TextTestRunner()
start_server()
result = runner.run(suite)
stop_server()
clear_test_environment()
if result.wasSuccessful():
sys.exit(0)
else:
sys.exit(1)
def main():
logging.basicConfig(level=logging.DEBUG)
parser = OptionParser()
parser.add_option('-t', '--test', help='Run only specified tests')
parser.add_option('--transport', default='pycurl')
parser.add_option('--test-grab', action='store_true',
default=False, help='Run tests for Grab::Spider')
parser.add_option('--test-spider', action='store_true',
default=False, help='Run tests for Grab')
parser.add_option('--test-all', action='store_true',
default=False,
help='Run tests for both Grab and Grab::Spider')
parser.add_option('--backend-mongo', action='store_true',
default=False,
help='Run extra tests that depends on mongodb')
parser.add_option('--backend-redis', action='store_true',
default=False,
help='Run extra tests that depends on redis')
parser.add_option('--backend-mysql', action='store_true',
default=False,
help='Run extra tests that depends on mysql')
parser.add_option('--backend-postgresql', action='store_true',
default=False,
help='Run extra tests that depends on postgresql')
parser.add_option('--mp-mode', action='store_true', default=False,
help='Enable multiprocess mode in spider tests')
parser.add_option('--profile', action='store_true', default=False,
help='Do profiling')
opts, args = parser.parse_args()
GLOBAL['transport'] = opts.transport
if opts.backend_mongo:
GLOBAL['backends'].append('mongo')
if opts.backend_redis:
GLOBAL['backends'].append('redis')
if opts.backend_mysql:
GLOBAL['backends'].append('mysql')
if opts.backend_postgresql:
GLOBAL['backends'].append('postgresql')
prepare_test_environment()
test_list = []
if opts.test_all:
test_list += GRAB_TEST_LIST
test_list += SPIDER_TEST_LIST
if opts.test_grab:
test_list += GRAB_TEST_LIST
if opts.test_spider:
test_list += SPIDER_TEST_LIST
if opts.test:
test_list += [opts.test]
GLOBAL['mp_mode'] = opts.mp_mode
# Check tests integrity
# Ensure that all test modules are imported correctly
for path in test_list:
__import__(path, None, None, ['foo'])
loader = unittest.TestLoader()
suite = unittest.TestSuite()
for path in test_list:
mod_suite = loader.loadTestsFromName(path)
for some_suite in mod_suite:
for test in some_suite:
if (not hasattr(test, '_backend') or
test._backend in GLOBAL['backends']):
suite.addTest(test)
runner = unittest.TextTestRunner()
#start_server()
if opts.profile:
import cProfile
import pyprof2calltree
import pstats
profile_tree_file = 'var/test.prof.out'
prof = cProfile.Profile()
result = prof.runcall(runner.run, suite)
stats = pstats.Stats(prof)
stats.strip_dirs()
pyprof2calltree.convert(stats, profile_tree_file)
else:
result = runner.run(suite)
clear_test_environment()
if result.wasSuccessful():
sys.exit(0)
else:
sys.exit(1)
def main():
logging.basicConfig(level=logging.DEBUG)
parser = OptionParser()
parser.add_option('-t', '--test', help='Run only specified tests')
parser.add_option('--test-grab',
action='store_true',
default=False,
help='Run tests for Grab::Spider')
parser.add_option('--test-spider',
action='store_true',
default=False,
help='Run tests for Grab')
parser.add_option('--test-all',
action='store_true',
default=False,
help='Run tests for both Grab and Grab::Spider')
parser.add_option('--backend-mongo',
action='store_true',
default=False,
help='Run extra tests that depends on mongodb')
parser.add_option('--backend-redis',
action='store_true',
default=False,
help='Run extra tests that depends on redis')
parser.add_option('--backend-mysql',
action='store_true',
default=False,
help='Run extra tests that depends on mysql')
parser.add_option('--backend-postgresql',
action='store_true',
default=False,
help='Run extra tests that depends on postgresql')
opts, args = parser.parse_args()
if opts.backend_mongo:
GLOBAL['backends'].append('mongo')
if opts.backend_redis:
GLOBAL['backends'].append('redis')
if opts.backend_mysql:
GLOBAL['backends'].append('mysql')
if opts.backend_postgresql:
GLOBAL['backends'].append('postgresql')
prepare_test_environment()
test_list = []
if opts.test_all:
test_list += GRAB_TEST_LIST
test_list += SPIDER_TEST_LIST
if opts.test_grab:
test_list += GRAB_TEST_LIST
if opts.test_spider:
test_list += SPIDER_TEST_LIST
if opts.test:
test_list += [opts.test]
# Check tests integrity
# Ensure that all test modules are imported correctly
for path in test_list:
__import__(path, None, None, ['foo'])
loader = unittest.TestLoader()
suite = unittest.TestSuite()
for path in test_list:
mod_suite = loader.loadTestsFromName(path)
for some_suite in mod_suite:
for test in some_suite:
if (not hasattr(test, '_backend')
or test._backend in GLOBAL['backends']):
suite.addTest(test)
runner = unittest.TextTestRunner()
result = runner.run(suite)
clear_test_environment()
if result.wasSuccessful():
sys.exit(0)
else:
sys.exit(1)
def main():
logging.basicConfig(level=logging.DEBUG)
parser = OptionParser()
parser.add_option("-t", "--test", help="Run only specified tests")
parser.add_option("--test-grab", action="store_true", default=False, help="Run tests for Grab::Spider")
parser.add_option("--test-spider", action="store_true", default=False, help="Run tests for Grab")
parser.add_option("--test-all", action="store_true", default=False, help="Run tests for both Grab and Grab::Spider")
parser.add_option(
"--backend-mongo", action="store_true", default=False, help="Run extra tests that depends on mongodb"
)
parser.add_option(
"--backend-redis", action="store_true", default=False, help="Run extra tests that depends on redis"
)
parser.add_option(
"--backend-mysql", action="store_true", default=False, help="Run extra tests that depends on mysql"
)
parser.add_option(
"--backend-postgresql", action="store_true", default=False, help="Run extra tests that depends on postgresql"
)
parser.add_option("--mp-mode", action="store_true", default=False, help="Enable multiprocess mode in spider tests")
opts, args = parser.parse_args()
if opts.backend_mongo:
GLOBAL["backends"].append("mongo")
if opts.backend_redis:
GLOBAL["backends"].append("redis")
if opts.backend_mysql:
GLOBAL["backends"].append("mysql")
if opts.backend_postgresql:
GLOBAL["backends"].append("postgresql")
prepare_test_environment()
test_list = []
if opts.test_all:
test_list += GRAB_TEST_LIST
test_list += SPIDER_TEST_LIST
if opts.test_grab:
test_list += GRAB_TEST_LIST
if opts.test_spider:
test_list += SPIDER_TEST_LIST
if opts.test:
test_list += [opts.test]
GLOBAL["mp_mode"] = opts.mp_mode
# Check tests integrity
# Ensure that all test modules are imported correctly
for path in test_list:
__import__(path, None, None, ["foo"])
loader = unittest.TestLoader()
suite = unittest.TestSuite()
for path in test_list:
mod_suite = loader.loadTestsFromName(path)
for some_suite in mod_suite:
for test in some_suite:
if not hasattr(test, "_backend") or test._backend in GLOBAL["backends"]:
suite.addTest(test)
runner = unittest.TextTestRunner()
# start_server()
result = runner.run(suite)
# stop_server()
clear_test_environment()
if result.wasSuccessful():
sys.exit(0)
else:
sys.exit(1)
def main():
logging.basicConfig(level=logging.DEBUG)
parser = OptionParser()
parser.add_option("-t", "--test", help="Run only specified tests")
parser.add_option("--transport", help="Test specified transport", default="grab.transport.curl.CurlTransport")
parser.add_option("--extra", action="store_true", default=False, help="Run extra tests for specific backends")
parser.add_option("--test-grab", action="store_true", default=False, help="Run tests for Grab::Spider")
parser.add_option("--test-spider", action="store_true", default=False, help="Run tests for Grab")
parser.add_option("--test-all", action="store_true", default=False, help="Run tests for both Grab and Grab::Spider")
parser.add_option("--test-kit", action="store_true", default=False, help="Run tests for Grab with WebKit transport")
opts, args = parser.parse_args()
test.util.GRAB_TRANSPORT = opts.transport
if opts.test_kit:
test.util.GRAB_TRANSPORT = "grab.transport.kit.KitTransport"
prepare_test_environment()
test_list = []
if opts.test_all:
test_list += GRAB_TEST_LIST
test_list += SPIDER_TEST_LIST
if opts.extra:
test_list += GRAB_EXTRA_TEST_LIST
test_list += SPIDER_EXTRA_TEST_LIST
if opts.test_grab:
test_list += GRAB_TEST_LIST
if opts.extra:
test_list += GRAB_EXTRA_TEST_LIST
if opts.test_kit:
test_list += KIT_TEST_LIST
if opts.extra:
test_list += KIT_EXTRA_TEST_LIST
if opts.test_spider:
test_list += SPIDER_TEST_LIST
if opts.extra:
test_list += SPIDER_EXTRA_TEST_LIST
if opts.test:
test_list += [opts.test]
# Check tests integrity
# Ensure that all test modules are imported correctly
for path in test_list:
__import__(path, None, None, ["foo"])
loader = unittest.TestLoader()
suite = loader.loadTestsFromNames(test_list)
runner = unittest.TextTestRunner()
start_server()
result = runner.run(suite)
stop_server()
clear_test_environment()
if result.wasSuccessful():
sys.exit(0)
else:
sys.exit(1)
