def setUpClass(self):
import tests.data_test_webpage
import httpbin
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run,
port=14887)
self.httpbin = 'http://127.0.0.1:14887'
self.inqueue = Queue(10)
self.outqueue = Queue(10)
self.fetcher = Fetcher(self.inqueue, self.outqueue)
self.fetcher.phantomjs_proxy = '127.0.0.1:25555'
self.rpc = xmlrpc_client.ServerProxy('http://localhost:%d' % 24444)
self.xmlrpc_thread = utils.run_in_thread(self.fetcher.xmlrpc_run,
port=24444)
self.thread = utils.run_in_thread(self.fetcher.run)
try:
self.phantomjs = subprocess.Popen([
'phantomjs',
os.path.join(os.path.dirname(__file__),
'../pyspider/fetcher/phantomjs_fetcher.js'),
'25555'
])
except OSError:
self.phantomjs = None
time.sleep(0.5)
def setUpClass(self):
import tests.data_test_webpage
import httpbin
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run,
port=14887,
passthrough_errors=False)
self.httpbin = 'http://127.0.0.1:14887'
self.inqueue = Queue(10)
self.outqueue = Queue(10)
self.fetcher = Fetcher(self.inqueue, self.outqueue)
self.fetcher.phantomjs_proxy = '127.0.0.1:25555'
self.rpc = xmlrpc_client.ServerProxy('http://localhost:%d' % 24444)
self.xmlrpc_thread = utils.run_in_thread(self.fetcher.xmlrpc_run,
port=24444)
self.thread = utils.run_in_thread(self.fetcher.run)
self.proxy_thread = subprocess.Popen([
'pyproxy', '--username=binux', '--password=123456', '--port=14830',
'--debug'
],
close_fds=True)
self.proxy = '127.0.0.1:14830'
try:
self.phantomjs = subprocess.Popen([
'phantomjs',
os.path.join(os.path.dirname(__file__),
'../pyspider/fetcher/phantomjs_fetcher.js'),
'25555'
])
except OSError:
self.phantomjs = None
time.sleep(0.5)
def setUpClass(self):
self.projectdb = ProjectDB([
os.path.join(os.path.dirname(__file__),
'data_fetcher_processor_handler.py')
])
self.fetcher = Fetcher(None, None, async=False)
self.status_queue = Queue()
self.newtask_queue = Queue()
self.result_queue = Queue()
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run,
port=14887,
passthrough_errors=False)
self.httpbin = 'http://127.0.0.1:14887'
self.proxy_thread = subprocess.Popen([
'pyproxy', '--username=binux', '--password=123456', '--port=14830',
'--debug'
],
close_fds=True)
self.proxy = '127.0.0.1:14830'
self.processor = Processor(projectdb=self.projectdb,
inqueue=None,
status_queue=self.status_queue,
newtask_queue=self.newtask_queue,
result_queue=self.result_queue)
self.project_name = 'data_fetcher_processor_handler'
time.sleep(0.5)
def setUpClass(self):
import tests.data_test_webpage
import httpbin
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887, passthrough_errors=False)
self.httpbin = 'http://127.0.0.1:14887'
self.inqueue = Queue(10)
self.outqueue = Queue(10)
self.fetcher = Fetcher(self.inqueue, self.outqueue)
self.fetcher.phantomjs_proxy = '127.0.0.1:25555'
self.rpc = xmlrpc_client.ServerProxy('http://localhost:%d' % 24444)
self.xmlrpc_thread = utils.run_in_thread(self.fetcher.xmlrpc_run, port=24444)
self.thread = utils.run_in_thread(self.fetcher.run)
self.proxy_thread = subprocess.Popen(['pyproxy', '--username=binux',
'--password=123456', '--port=14830',
'--debug'], close_fds=True)
self.proxy = '127.0.0.1:14830'
try:
self.phantomjs = subprocess.Popen(['phantomjs',
os.path.join(os.path.dirname(__file__),
'../pyspider/fetcher/phantomjs_fetcher.js'),
'25555'])
except OSError:
self.phantomjs = None
time.sleep(0.5)
def setUpClass(self):
import tests.data_test_webpage
import httpbin
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run,
host='0.0.0.0',
port=14887,
passthrough_errors=False)
self.httpbin = 'http://' + socket.gethostbyname(
socket.gethostname()) + ':14887'
self.inqueue = Queue(10)
self.outqueue = Queue(10)
self.fetcher = Fetcher(self.inqueue, self.outqueue)
self.fetcher.splash_endpoint = 'http://127.0.0.1:8050/execute'
self.rpc = xmlrpc_client.ServerProxy('http://localhost:%d' % 24444)
self.xmlrpc_thread = utils.run_in_thread(self.fetcher.xmlrpc_run,
port=24444)
self.thread = utils.run_in_thread(self.fetcher.run)
self.proxy_thread = subprocess.Popen([
'pyproxy', '--username=binux', '--bind=0.0.0.0',
'--password=123456', '--port=14830', '--debug'
],
close_fds=True)
self.proxy = '127.0.0.1:14830'
def setUpClass(self):
self.fetcher = Fetcher(None, None, async_mode=False)
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run,
port=14887,
passthrough_errors=False)
self.httpbin = 'http://127.0.0.1:14887'
time.sleep(0.5)
def setUpClass(self):
import tests.data_test_webpage
import httpbin
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887)
self.httpbin = "http://127.0.0.1:14887"
self.inqueue = Queue(10)
self.outqueue = Queue(10)
self.fetcher = Fetcher(self.inqueue, self.outqueue)
self.fetcher.phantomjs_proxy = "127.0.0.1:25555"
self.rpc = xmlrpc_client.ServerProxy("http://localhost:%d" % 24444)
self.xmlrpc_thread = utils.run_in_thread(self.fetcher.xmlrpc_run, port=24444)
self.thread = utils.run_in_thread(self.fetcher.run)
self.proxy_thread = subprocess.Popen(
["pyproxy", "--username=binux", "--password=123456", "--port=14830", "--debug"], close_fds=True
)
self.proxy = "127.0.0.1:14830"
try:
self.phantomjs = subprocess.Popen(
[
"phantomjs",
os.path.join(os.path.dirname(__file__), "../pyspider/fetcher/phantomjs_fetcher.js"),
"25555",
]
)
except OSError:
self.phantomjs = None
time.sleep(0.5)
def setUpClass(self):
shutil.rmtree('./data/tests', ignore_errors=True)
os.makedirs('./data/tests')
import tests.data_test_webpage
import httpbin
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887, passthrough_errors=False)
self.httpbin = 'http://127.0.0.1:14887'
def setUpClass(self):
shutil.rmtree('./data/tests', ignore_errors=True)
os.makedirs('./data/tests')
import tests.data_test_webpage
import httpbin
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887)
self.httpbin = 'http://127.0.0.1:14887'
def setUpClass(self):
shutil.rmtree('./data/tests', ignore_errors=True)
os.makedirs('./data/tests')
import tests.data_test_webpage
import httpbin
from pyspider.webui import bench_test # flake8: noqa
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run,
port=14887,
passthrough_errors=False)
self.httpbin = 'http://127.0.0.1:14887'
ctx = run.cli.make_context('test', [
'--taskdb',
'sqlalchemy+sqlite+taskdb:///data/tests/task.db',
'--projectdb',
'sqlalchemy+sqlite+projectdb:///data/tests/projectdb.db',
'--resultdb',
'sqlalchemy+sqlite+resultdb:///data/tests/resultdb.db',
],
None,
obj=ObjectDict(testing_mode=True))
self.ctx = run.cli.invoke(ctx)
self.threads = []
ctx = run.scheduler.make_context('scheduler', [], self.ctx)
self.scheduler = scheduler = run.scheduler.invoke(ctx)
self.threads.append(run_in_thread(scheduler.xmlrpc_run))
self.threads.append(run_in_thread(scheduler.run))
ctx = run.fetcher.make_context('fetcher', [
'--xmlrpc',
'--xmlrpc-port',
'24444',
], self.ctx)
fetcher = run.fetcher.invoke(ctx)
self.threads.append(run_in_thread(fetcher.xmlrpc_run))
self.threads.append(run_in_thread(fetcher.run))
ctx = run.processor.make_context('processor', [], self.ctx)
processor = run.processor.invoke(ctx)
self.threads.append(run_in_thread(processor.run))
ctx = run.result_worker.make_context('result_worker', [], self.ctx)
result_worker = run.result_worker.invoke(ctx)
self.threads.append(run_in_thread(result_worker.run))
ctx = run.webui.make_context(
'webui', ['--scheduler-rpc', 'http://localhost:23333/'], self.ctx)
app = run.webui.invoke(ctx)
app.debug = True
self.app = app.test_client()
self.rpc = app.config['scheduler_rpc']
time.sleep(1)
def setUpClass(self):
shutil.rmtree('./data/tests', ignore_errors=True)
os.makedirs('./data/tests')
import tests.data_test_webpage
import httpbin
from pyspider.webui import bench_test # flake8: noqa
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887, passthrough_errors=False)
self.httpbin = 'http://127.0.0.1:14887'
ctx = run.cli.make_context('test', [
'--taskdb', 'sqlalchemy+sqlite+taskdb:///data/tests/task.db',
'--projectdb', 'sqlalchemy+sqlite+projectdb:///data/tests/projectdb.db',
'--resultdb', 'sqlalchemy+sqlite+resultdb:///data/tests/resultdb.db',
], None, obj=ObjectDict(testing_mode=True))
self.ctx = run.cli.invoke(ctx)
self.threads = []
ctx = run.scheduler.make_context('scheduler', [], self.ctx)
self.scheduler = scheduler = run.scheduler.invoke(ctx)
self.threads.append(run_in_thread(scheduler.xmlrpc_run))
self.threads.append(run_in_thread(scheduler.run))
ctx = run.fetcher.make_context('fetcher', [
'--xmlrpc',
'--xmlrpc-port', '24444',
], self.ctx)
fetcher = run.fetcher.invoke(ctx)
self.threads.append(run_in_thread(fetcher.xmlrpc_run))
self.threads.append(run_in_thread(fetcher.run))
ctx = run.processor.make_context('processor', [], self.ctx)
processor = run.processor.invoke(ctx)
self.threads.append(run_in_thread(processor.run))
ctx = run.result_worker.make_context('result_worker', [], self.ctx)
result_worker = run.result_worker.invoke(ctx)
self.threads.append(run_in_thread(result_worker.run))
ctx = run.webui.make_context('webui', [
'--scheduler-rpc', 'http://localhost:23333/'
], self.ctx)
app = run.webui.invoke(ctx)
app.debug = True
self.app = app.test_client()
self.rpc = app.config['scheduler_rpc']
time.sleep(1)
def setUpClass(self):
shutil.rmtree("./data/tests", ignore_errors=True)
os.makedirs("./data/tests")
import tests.data_test_webpage
import httpbin
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887)
self.httpbin = "http://127.0.0.1:14887"
ctx = run.cli.make_context(
"test",
[
"--taskdb",
"sqlite+taskdb:///data/tests/task.db",
"--projectdb",
"sqlite+projectdb:///data/tests/projectdb.db",
"--resultdb",
"sqlite+resultdb:///data/tests/resultdb.db",
],
None,
obj=ObjectDict(testing_mode=True),
)
self.ctx = run.cli.invoke(ctx)
ctx = run.scheduler.make_context("scheduler", [], self.ctx)
scheduler = run.scheduler.invoke(ctx)
run_in_thread(scheduler.xmlrpc_run)
run_in_thread(scheduler.run)
ctx = run.fetcher.make_context("fetcher", [], self.ctx)
fetcher = run.fetcher.invoke(ctx)
run_in_thread(fetcher.run)
ctx = run.processor.make_context("processor", [], self.ctx)
processor = run.processor.invoke(ctx)
run_in_thread(processor.run)
ctx = run.result_worker.make_context("result_worker", [], self.ctx)
result_worker = run.result_worker.invoke(ctx)
run_in_thread(result_worker.run)
ctx = run.webui.make_context("webui", ["--scheduler-rpc", "http://localhost:23333/"], self.ctx)
app = run.webui.invoke(ctx)
app.debug = True
self.app = app.test_client()
self.rpc = app.config["scheduler_rpc"]
time.sleep(1)
def setUpClass(self):
import tests.data_test_webpage
import httpbin
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, host='0.0.0.0', port=14887, passthrough_errors=False)
self.httpbin = 'http://' + socket.gethostbyname(socket.gethostname()) + ':14887'
self.inqueue = Queue(10)
self.outqueue = Queue(10)
self.fetcher = Fetcher(self.inqueue, self.outqueue)
self.fetcher.splash_endpoint = 'http://127.0.0.1:8050/execute'
self.rpc = xmlrpc_client.ServerProxy('http://localhost:%d' % 24444)
self.xmlrpc_thread = utils.run_in_thread(self.fetcher.xmlrpc_run, port=24444)
self.thread = utils.run_in_thread(self.fetcher.run)
self.proxy_thread = subprocess.Popen(['pyproxy', '--username=binux',
'--password=123456', '--port=14830',
'--debug'], close_fds=True)
self.proxy = '127.0.0.1:14830'
def setUpClass(self):
self.projectdb = ProjectDB([os.path.join(os.path.dirname(__file__), 'data_fetcher_processor_handler.py')])
self.fetcher = Fetcher(None, None, async=False)
self.status_queue = Queue()
self.newtask_queue = Queue()
self.result_queue = Queue()
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887)
self.httpbin = 'http://127.0.0.1:14887'
self.proxy_thread = subprocess.Popen(['pyproxy', '--username=binux',
'--password=123456', '--port=14830',
'--debug'], close_fds=True)
self.proxy = '127.0.0.1:14830'
self.processor = Processor(projectdb=self.projectdb,
inqueue=None,
status_queue=self.status_queue,
newtask_queue=self.newtask_queue,
result_queue=self.result_queue)
self.project_name = 'data_fetcher_processor_handler'
time.sleep(0.5)
def setUpClass(self):
self.projectdb = ProjectDB([
os.path.join(os.path.dirname(__file__),
'data_fetcher_processor_handler.py')
])
self.fetcher = Fetcher(None, None, async=False)
self.status_queue = Queue()
self.newtask_queue = Queue()
self.result_queue = Queue()
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run,
port=14887)
self.httpbin = 'http://127.0.0.1:14887'
self.processor = Processor(projectdb=self.projectdb,
inqueue=None,
status_queue=self.status_queue,
newtask_queue=self.newtask_queue,
result_queue=self.result_queue)
self.project_name = 'data_fetcher_processor_handler'
time.sleep(0.5)
def setUpClass(self):
self.fetcher = Fetcher(None, None, async=False)
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887, passthrough_errors=False)
self.httpbin = 'http://127.0.0.1:14887'
time.sleep(0.5)
