def setUpClass(self):
shutil.rmtree("./data/tests", ignore_errors=True)
os.makedirs("./data/tests")
run_in_thread(run.run_scheduler, g=run.g)
run_in_thread(run.run_fetcher, g=run.g)
run_in_thread(run.run_processor, g=run.g)
run_in_thread(run.run_result_worker, g=run.g)
time.sleep(1)
app.config["taskdb"] = run.g.taskdb
app.config["projectdb"] = run.g.projectdb
app.config["resultdb"] = run.g.resultdb
app.config["scheduler_rpc"] = xmlrpclib.ServerProxy("http://localhost:23333")
self.app = app.test_client()
def setUpClass(self):
shutil.rmtree('./data/tests', ignore_errors=True)
os.makedirs('./data/tests')
run_in_thread(run.run_scheduler, g=run.g)
run_in_thread(run.run_fetcher, g=run.g)
run_in_thread(run.run_processor, g=run.g)
run_in_thread(run.run_result_worker, g=run.g)
time.sleep(1)
app.config['taskdb'] = run.g.taskdb
app.config['projectdb'] = run.g.projectdb
app.config['resultdb'] = run.g.resultdb
app.config['scheduler_rpc'] = xmlrpclib.ServerProxy(
'http://localhost:23333')
self.app = app.test_client()
def setUpClass(self):
shutil.rmtree('./data/tests', ignore_errors=True)
os.makedirs('./data/tests')
ctx = run.cli.make_context('test', [
'--taskdb', 'sqlite+taskdb:///data/tests/task.db',
'--projectdb', 'sqlite+projectdb:///data/tests/projectdb.db',
'--resultdb', 'sqlite+resultdb:///data/tests/resultdb.db',
], None, obj=ObjectDict(testing_mode=True))
self.ctx = run.cli.invoke(ctx)
ctx = run.scheduler.make_context('scheduler', [], self.ctx)
scheduler = run.scheduler.invoke(ctx)
run_in_thread(scheduler.xmlrpc_run)
run_in_thread(scheduler.run)
ctx = run.fetcher.make_context('fetcher', [], self.ctx)
fetcher = run.fetcher.invoke(ctx)
run_in_thread(fetcher.run)
ctx = run.processor.make_context('processor', [], self.ctx)
processor = run.processor.invoke(ctx)
run_in_thread(processor.run)
ctx = run.result_worker.make_context('result_worker', [], self.ctx)
result_worker = run.result_worker.invoke(ctx)
run_in_thread(result_worker.run)
ctx = run.webui.make_context('webui', [
'--scheduler-rpc', 'http://localhost:23333/'
], self.ctx)
app = run.webui.invoke(ctx)
self.app = app.test_client()
self.rpc = app.config['scheduler_rpc']
time.sleep(1)
def bench(ctx, fetcher_num, processor_num, result_worker_num, run_in, total, show):
from pyspider.libs import bench
from pyspider.webui import bench_test
ctx.obj['debug'] = False
g = ctx.obj
if result_worker_num == 0:
g['processor2result'] = None
if run_in == 'subprocess' and os.name != 'nt':
run_in = run_in_subprocess
else:
run_in = run_in_thread
g.projectdb.insert('bench', {
'name': 'bench',
'status': 'RUNNING',
'script': bench.bench_script % {'total': total, 'show': show},
'rate': 100000000000000,
'burst': 10000000000000000,
'updatetime': time.time()
})
# disable log
logging.getLogger().setLevel(logging.ERROR)
logging.getLogger('scheduler').setLevel(logging.ERROR)
logging.getLogger('fetcher').setLevel(logging.ERROR)
logging.getLogger('processor').setLevel(logging.ERROR)
logging.getLogger('result').setLevel(logging.ERROR)
logging.getLogger('webui').setLevel(logging.ERROR)
threads = []
# result worker
result_worker_config = g.config.get('result_worker', {})
for i in range(result_worker_num):
threads.append(run_in(ctx.invoke, result_worker,
ResultWorker=bench.BenchResultWorker, **result_worker_config))
# processor
processor_config = g.config.get('processor', {})
for i in range(processor_num):
threads.append(run_in(ctx.invoke, processor,
Processor=bench.BenchProcessor, **processor_config))
# fetcher
fetcher_config = g.config.get('fetcher', {})
fetcher_config.setdefault('xmlrpc_host', '127.0.0.1')
for i in range(fetcher_num):
threads.append(run_in(ctx.invoke, fetcher,
Fetcher=bench.BenchFetcher, **fetcher_config))
# scheduler
scheduler_config = g.config.get('scheduler', {})
scheduler_config.setdefault('xmlrpc_host', '127.0.0.1')
threads.append(run_in(ctx.invoke, scheduler,
Scheduler=bench.BenchScheduler, **scheduler_config))
# running webui in main thread to make it exitable
webui_config = g.config.get('webui', {})
webui_config.setdefault('scheduler_rpc', 'http://localhost:%s/'
% g.config.get('scheduler', {}).get('xmlrpc_port', 23333))
g['testing_mode'] = True
app = ctx.invoke(webui, **webui_config)
# run project
app_client = app.test_client()
rv = app_client.post('/run', data={
'project': 'bench',
})
assert rv.status_code == 200, 'run project error'
app.run('127.0.0.1', 5000)
for each in g.instances:
each.quit()
for each in threads:
each.join()
