def setUpClass(self): import tests.data_test_webpage import httpbin self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887, passthrough_errors=False) self.httpbin = 'http://127.0.0.1:14887' self.inqueue = Queue(10) self.outqueue = Queue(10) self.fetcher = Fetcher(self.inqueue, self.outqueue) self.fetcher.phantomjs_proxy = '127.0.0.1:25555' self.rpc = xmlrpc_client.ServerProxy('http://localhost:%d' % 24444) self.xmlrpc_thread = utils.run_in_thread(self.fetcher.xmlrpc_run, port=24444) self.thread = utils.run_in_thread(self.fetcher.run) self.proxy_thread = subprocess.Popen(['pyproxy', '--username=binux', '--password=123456', '--port=14830', '--debug'], close_fds=True) self.proxy = '127.0.0.1:14830' try: self.phantomjs = subprocess.Popen(['phantomjs', os.path.join(os.path.dirname(__file__), '../pyspider/fetcher/phantomjs_fetcher.js'), '25555']) except OSError: self.phantomjs = None time.sleep(0.5)
def setUpClass(self): self.inqueue = Queue(10) self.outqueue = Queue(10) self.fetcher = Fetcher(self.inqueue, self.outqueue) self.rpc = xmlrpclib.ServerProxy('http://localhost:%d' % 24444) self.xmlrpc_thread = utils.run_in_thread(self.fetcher.xmlrpc_run, port=24444) self.thread = utils.run_in_thread(self.fetcher.run)
def scheduler( ctx, xmlrpc, xmlrpc_host, xmlrpc_port, inqueue_limit, delete_time, active_tasks, loop_limit, scheduler_cls ): g = ctx.obj Scheduler = load_cls(None, None, scheduler_cls) scheduler = Scheduler( taskdb=g.taskdb, projectdb=g.projectdb, resultdb=g.resultdb, newtask_queue=g.newtask_queue, status_queue=g.status_queue, out_queue=g.scheduler2fetcher, data_path=g.get("data_path", "data"), ) scheduler.INQUEUE_LIMIT = inqueue_limit scheduler.DELETE_TIME = delete_time scheduler.ACTIVE_TASKS = active_tasks scheduler.LOOP_LIMIT = loop_limit g.instances.append(scheduler) if g.get("testing_mode"): return scheduler if xmlrpc: utils.run_in_thread(scheduler.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host) scheduler.run()
def bloomfilter(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, key, capacity, error, redis): """ Run bloomfilter, only one bloomfilter is allowed. """ g = ctx.obj if os.name == 'nt': from pyspider.filter import BloomFilter bloomfilter = BloomFilter(key, capacity, error) else: from pyspider.filter import RedisBloomFilter from six.moves.urllib.parse import urlparse parsed = urlparse(url) # ParseResult(scheme='', netloc='127.0.0.1:6379', path='/0', params='', query='', fragment='') bloomfilter = RedisBloomFilter(key, capacity, error, parsed.hostname, parsed.port, int(parsed.path.strip('/') or 0)) g.instances.append(bloomfilter) if g.get('testing_mode'): return bloomfilter if xmlrpc: utils.run_in_thread(bloomfilter.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host) bloomfilter.run()
def scheduler(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, inqueue_limit, delete_time, active_tasks, loop_limit, scheduler_cls, threads): """ Run Scheduler, only one scheduler is allowed. """ g = ctx.obj Scheduler = load_cls(None, None, scheduler_cls) kwargs = dict(taskdb=g.taskdb, projectdb=g.projectdb, resultdb=g.resultdb, newtask_queue=g.newtask_queue, status_queue=g.status_queue, out_queue=g.scheduler2fetcher, data_path=g.get('data_path', 'data')) if threads: kwargs['threads'] = int(threads) scheduler = Scheduler(**kwargs) scheduler.INQUEUE_LIMIT = inqueue_limit scheduler.DELETE_TIME = delete_time scheduler.ACTIVE_TASKS = active_tasks scheduler.LOOP_LIMIT = loop_limit g.instances.append(scheduler) if g.get('testing_mode'): return scheduler if xmlrpc: utils.run_in_thread(scheduler.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host) scheduler.run()
def setUpClass(self): import tests.data_test_webpage import httpbin self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887) self.httpbin = "http://127.0.0.1:14887" self.inqueue = Queue(10) self.outqueue = Queue(10) self.fetcher = Fetcher(self.inqueue, self.outqueue) self.fetcher.phantomjs_proxy = "127.0.0.1:25555" self.rpc = xmlrpc_client.ServerProxy("http://localhost:%d" % 24444) self.xmlrpc_thread = utils.run_in_thread(self.fetcher.xmlrpc_run, port=24444) self.thread = utils.run_in_thread(self.fetcher.run) self.proxy_thread = subprocess.Popen( ["pyproxy", "--username=binux", "--password=123456", "--port=14830", "--debug"], close_fds=True ) self.proxy = "127.0.0.1:14830" try: self.phantomjs = subprocess.Popen( [ "phantomjs", os.path.join(os.path.dirname(__file__), "../pyspider/fetcher/phantomjs_fetcher.js"), "25555", ] ) except OSError: self.phantomjs = None time.sleep(0.5)
def run_fetcher(g=g): from pyspider.fetcher.tornado_fetcher import Fetcher fetcher = Fetcher(inqueue=g.scheduler2fetcher, outqueue=g.fetcher2processor) fetcher.phantomjs_proxy = g.phantomjs_proxy run_in_thread(fetcher.xmlrpc_run, port=g.fetcher_xmlrpc_port, bind=g.webui_host) fetcher.run()
def setUpClass(self): shutil.rmtree("./data/tests", ignore_errors=True) os.makedirs("./data/tests") ctx = run.cli.make_context( "test", [ "--taskdb", "sqlite+taskdb:///data/tests/task.db", "--projectdb", "sqlite+projectdb:///data/tests/projectdb.db", "--resultdb", "sqlite+resultdb:///data/tests/resultdb.db", ], None, obj=dict(testing_mode=True), ) self.ctx = run.cli.invoke(ctx) ctx = run.scheduler.make_context("scheduler", [], self.ctx) scheduler = run.scheduler.invoke(ctx) utils.run_in_thread(scheduler.xmlrpc_run) utils.run_in_thread(scheduler.run) time.sleep(1)
def fetcher(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, poolsize, proxy, user_agent, timeout, phantomjs_endpoint, splash_endpoint, fetcher_cls, async_mode=True, get_object=False, no_input=False): """ Run Fetcher. """ g = ctx.obj Fetcher = load_cls(None, None, fetcher_cls) if no_input: inqueue = None outqueue = None else: inqueue = g.scheduler2fetcher outqueue = g.fetcher2processor fetcher = Fetcher(inqueue=inqueue, outqueue=outqueue, poolsize=poolsize, proxy=proxy, async_mode=async_mode) fetcher.phantomjs_proxy = phantomjs_endpoint or g.phantomjs_proxy fetcher.splash_endpoint = splash_endpoint if user_agent: fetcher.user_agent = user_agent if timeout: fetcher.default_options = copy.deepcopy(fetcher.default_options) fetcher.default_options['timeout'] = timeout g.instances.append(fetcher) if g.get('testing_mode') or get_object: return fetcher if xmlrpc: utils.run_in_thread(fetcher.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host) fetcher.run()
def run_scheduler(g=g): from pyspider.scheduler import Scheduler scheduler = Scheduler(taskdb=g.taskdb, projectdb=g.projectdb, resultdb=g.resultdb, newtask_queue=g.newtask_queue, status_queue=g.status_queue, out_queue=g.scheduler2fetcher) g.scheduler = scheduler run_in_thread(scheduler.xmlrpc_run) scheduler.run()
def fetcher(ctx, xmlrpc, xmlrpc_host, xmlrpc_port): g = ctx.obj from pyspider.fetcher.tornado_fetcher import Fetcher fetcher = Fetcher(inqueue=g.scheduler2fetcher, outqueue=g.fetcher2processor) fetcher.phantomjs_proxy = g.phantomjs_proxy g.instances.append(fetcher) if xmlrpc: run_in_thread(fetcher.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host) fetcher.run()
def run_scheduler(g=g): from pyspider.scheduler import Scheduler scheduler = Scheduler(taskdb=g.taskdb, projectdb=g.projectdb, resultdb=g.resultdb, newtask_queue=g.newtask_queue, status_queue=g.status_queue, out_queue=g.scheduler2fetcher) if g.demo_mode: scheduler.INQUEUE_LIMIT = 1000 run_in_thread(scheduler.xmlrpc_run, port=g.scheduler_xmlrpc_port, bind=g.webui_host) scheduler.run()
def test_40_multiple_threading_error(self): def put(q): for i in range(100): q.put("DATA_%d" % i) def get(q): for i in range(100): q.get() utils.run_in_thread(put, self.q3) get(self.q3)
def setUpClass(self): self.inqueue = Queue(10) self.outqueue = Queue(10) self.fetcher = Fetcher(self.inqueue, self.outqueue) self.fetcher.phantomjs_proxy = 'localhost:25555' self.rpc = xmlrpclib.ServerProxy('http://localhost:%d' % 24444) self.xmlrpc_thread = utils.run_in_thread(self.fetcher.xmlrpc_run, port=24444) self.thread = utils.run_in_thread(self.fetcher.run) self.phantomjs = subprocess.Popen(['phantomjs', os.path.join(os.path.dirname(__file__), '../pyspider/fetcher/phantomjs_fetcher.js'), '25555'])
def run_scheduler(): scheduler = Scheduler(taskdb=get_taskdb(), projectdb=get_projectdb(), newtask_queue=self.newtask_queue, status_queue=self.status_queue, out_queue=self.scheduler2fetcher, data_path="./data/tests/", resultdb=get_resultdb()) scheduler.UPDATE_PROJECT_INTERVAL = 0.1 scheduler.LOOP_INTERVAL = 0.1 scheduler.INQUEUE_LIMIT = 10 Scheduler.DELETE_TIME = 0 scheduler._last_tick = int(time.time()) # not dispatch cronjob run_in_thread(scheduler.xmlrpc_run, port=self.scheduler_xmlrpc_port) scheduler.run()
def setUpClass(self): shutil.rmtree("./data/tests", ignore_errors=True) os.makedirs("./data/tests") import tests.data_test_webpage import httpbin self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887) self.httpbin = "http://127.0.0.1:14887" ctx = run.cli.make_context( "test", [ "--taskdb", "sqlite+taskdb:///data/tests/task.db", "--projectdb", "sqlite+projectdb:///data/tests/projectdb.db", "--resultdb", "sqlite+resultdb:///data/tests/resultdb.db", ], None, obj=ObjectDict(testing_mode=True), ) self.ctx = run.cli.invoke(ctx) ctx = run.scheduler.make_context("scheduler", [], self.ctx) scheduler = run.scheduler.invoke(ctx) run_in_thread(scheduler.xmlrpc_run) run_in_thread(scheduler.run) ctx = run.fetcher.make_context("fetcher", [], self.ctx) fetcher = run.fetcher.invoke(ctx) run_in_thread(fetcher.run) ctx = run.processor.make_context("processor", [], self.ctx) processor = run.processor.invoke(ctx) run_in_thread(processor.run) ctx = run.result_worker.make_context("result_worker", [], self.ctx) result_worker = run.result_worker.invoke(ctx) run_in_thread(result_worker.run) ctx = run.webui.make_context("webui", ["--scheduler-rpc", "http://localhost:23333/"], self.ctx) app = run.webui.invoke(ctx) app.debug = True self.app = app.test_client() self.rpc = app.config["scheduler_rpc"] time.sleep(1)
def scheduler(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, inqueue_limit, delete_time, active_tasks): g = ctx.obj from pyspider.scheduler import Scheduler scheduler = Scheduler(taskdb=g.taskdb, projectdb=g.projectdb, resultdb=g.resultdb, newtask_queue=g.newtask_queue, status_queue=g.status_queue, out_queue=g.scheduler2fetcher) scheduler.INQUEUE_LIMIT = inqueue_limit scheduler.DELETE_TIME = delete_time scheduler.ACTIVE_TASKS = active_tasks g.instances.append(scheduler) if xmlrpc: run_in_thread(scheduler.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host) scheduler.run()
def setUpClass(self): shutil.rmtree('./data/tests', ignore_errors=True) os.makedirs('./data/tests') def get_taskdb(): return taskdb.TaskDB(self.taskdb_path) self.taskdb = get_taskdb() def get_projectdb(): return projectdb.ProjectDB(self.projectdb_path) self.projectdb = get_projectdb() def get_resultdb(): return resultdb.ResultDB(self.resultdb_path) self.resultdb = get_resultdb() self.newtask_queue = Queue(10) self.status_queue = Queue(10) self.scheduler2fetcher = Queue(10) self.rpc = xmlrpclib.ServerProxy('http://localhost:%d' % self.scheduler_xmlrpc_port) def run_scheduler(): scheduler = Scheduler(taskdb=get_taskdb(), projectdb=get_projectdb(), newtask_queue=self.newtask_queue, status_queue=self.status_queue, out_queue=self.scheduler2fetcher, data_path="./data/tests/", resultdb=get_resultdb()) scheduler.UPDATE_PROJECT_INTERVAL = 0.1 scheduler.LOOP_INTERVAL = 0.1 scheduler.INQUEUE_LIMIT = 10 Scheduler.DELETE_TIME = 0 scheduler._last_tick = int(time.time()) # not dispatch cronjob run_in_thread(scheduler.xmlrpc_run, port=self.scheduler_xmlrpc_port) scheduler.run() self.process = run_in_thread(run_scheduler) time.sleep(1)
def setUpClass(self): import easywebdav shutil.rmtree('./data/tests', ignore_errors=True) os.makedirs('./data/tests') ctx = run.cli.make_context('test', [ '--taskdb', 'sqlite+taskdb:///data/tests/task.db', '--projectdb', 'sqlite+projectdb:///data/tests/projectdb.db', '--resultdb', 'sqlite+resultdb:///data/tests/resultdb.db', ], None, obj=utils.ObjectDict(testing_mode=True)) self.ctx = run.cli.invoke(ctx) ctx = run.webui.make_context('webui', [ '--username', 'binux', '--password', '4321', '--need-auth', ], self.ctx) self.app = run.webui.invoke(ctx) self.app_thread = utils.run_in_thread(self.app.run) time.sleep(5) self.webdav = easywebdav.connect('localhost', port=5000, path='dav') self.webdav_up = easywebdav.connect('localhost', port=5000, path='dav', username='******', password='******')
def setUpClass(self): shutil.rmtree('./data/tests', ignore_errors=True) os.makedirs('./data/tests') ctx = run.cli.make_context('test', [ '--taskdb', 'sqlite+taskdb:///data/tests/task.db', '--projectdb', 'sqlite+projectdb:///data/tests/projectdb.db', '--resultdb', 'sqlite+resultdb:///data/tests/resultdb.db', ], None, obj=dict(testing_mode=True)) self.ctx = run.cli.invoke(ctx) ctx = run.scheduler.make_context('scheduler', [], self.ctx) scheduler = run.scheduler.invoke(ctx) utils.run_in_thread(scheduler.xmlrpc_run) utils.run_in_thread(scheduler.run) time.sleep(1)
def scheduler(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, inqueue_limit, delete_time, active_tasks, Scheduler=Scheduler): g = ctx.obj scheduler = Scheduler(taskdb=g.taskdb, projectdb=g.projectdb, resultdb=g.resultdb, newtask_queue=g.newtask_queue, status_queue=g.status_queue, out_queue=g.scheduler2fetcher, data_path=g.get('data_path', 'data')) scheduler.INQUEUE_LIMIT = inqueue_limit scheduler.DELETE_TIME = delete_time scheduler.ACTIVE_TASKS = active_tasks g.instances.append(scheduler) if g.get('testing_mode'): return scheduler if xmlrpc: run_in_thread(scheduler.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host) scheduler.run()
def setUpClass(self): import tests.data_test_webpage import httpbin self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, host='0.0.0.0', port=14887, passthrough_errors=False) self.httpbin = 'http://' + socket.gethostbyname(socket.gethostname()) + ':14887' self.inqueue = Queue(10) self.outqueue = Queue(10) self.fetcher = Fetcher(self.inqueue, self.outqueue) self.fetcher.splash_endpoint = 'http://127.0.0.1:8050/execute' self.rpc = xmlrpc_client.ServerProxy('http://localhost:%d' % 24444) self.xmlrpc_thread = utils.run_in_thread(self.fetcher.xmlrpc_run, port=24444) self.thread = utils.run_in_thread(self.fetcher.run) self.proxy_thread = subprocess.Popen(['pyproxy', '--username=binux', '--password=123456', '--port=14830', '--debug'], close_fds=True) self.proxy = '127.0.0.1:14830'
def fetcher(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, poolsize, proxy, user_agent, timeout, Fetcher=Fetcher): g = ctx.obj fetcher = Fetcher(inqueue=g.scheduler2fetcher, outqueue=g.fetcher2processor, poolsize=poolsize, proxy=proxy) fetcher.phantomjs_proxy = g.phantomjs_proxy if user_agent: fetcher.user_agent = user_agent if timeout: fetcher.default_options = dict(fetcher.default_options) fetcher.default_options['timeout'] = timeout g.instances.append(fetcher) if g.get('testing_mode'): return fetcher if xmlrpc: run_in_thread(fetcher.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host) fetcher.run()
def fetcher(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, poolsize, proxy, user_agent, timeout, fetcher_cls): g = ctx.obj Fetcher = load_cls(None, None, fetcher_cls) fetcher = Fetcher(inqueue=g.scheduler2fetcher, outqueue=g.fetcher2processor, poolsize=poolsize, proxy=proxy) fetcher.phantomjs_proxy = g.phantomjs_proxy if user_agent: fetcher.user_agent = user_agent if timeout: fetcher.default_options = copy.deepcopy(fetcher.default_options) fetcher.default_options["timeout"] = timeout g.instances.append(fetcher) if g.get("testing_mode"): return fetcher if xmlrpc: utils.run_in_thread(fetcher.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host) fetcher.run()
def setUpClass(self): shutil.rmtree('./data/tests', ignore_errors=True) os.makedirs('./data/tests') import tests.data_test_webpage import httpbin self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887) self.httpbin = 'http://127.0.0.1:14887' ctx = run.cli.make_context('test', [ '--taskdb', 'sqlite+taskdb:///data/tests/task.db', '--projectdb', 'sqlite+projectdb:///data/tests/projectdb.db', '--resultdb', 'sqlite+resultdb:///data/tests/resultdb.db', ], None, obj=ObjectDict(testing_mode=True)) self.ctx = run.cli.invoke(ctx) ctx = run.scheduler.make_context('scheduler', [], self.ctx) scheduler = run.scheduler.invoke(ctx) run_in_thread(scheduler.xmlrpc_run) run_in_thread(scheduler.run) ctx = run.fetcher.make_context('fetcher', [], self.ctx) fetcher = run.fetcher.invoke(ctx) run_in_thread(fetcher.run) ctx = run.processor.make_context('processor', [], self.ctx) processor = run.processor.invoke(ctx) run_in_thread(processor.run) ctx = run.result_worker.make_context('result_worker', [], self.ctx) result_worker = run.result_worker.invoke(ctx) run_in_thread(result_worker.run) ctx = run.webui.make_context('webui', [ '--scheduler-rpc', 'http://localhost:23333/' ], self.ctx) app = run.webui.invoke(ctx) app.debug = True self.app = app.test_client() self.rpc = app.config['scheduler_rpc'] time.sleep(1)
def test_60_timeout_in_thread(self): base_task = self.base_task fetch_result = self.fetch_result base_task['process']['callback'] = 'sleep' base_task['process']['process_time_limit'] = 0.5 fetch_result['save'] = 2 start_time = time.time() thread = utils.run_in_thread(lambda self=self: self.instance.run_task(self.module, base_task, fetch_result)) thread.join() self.assertGreaterEqual(time.time() - start_time, 2)
def test_40_multiple_threading_error(self): def put(q): for i in range(100): q.put("DATA_{0:d}".format(i)) def get(q): for i in range(100): q.get() t = utils.run_in_thread(put, self.q3) get(self.q3) t.join()
def scheduler(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, inqueue_limit, delete_time, active_tasks, loop_limit, scheduler_cls, threads, bloomfilter_on, bloomfilter_rpc, get_object=False): """ Run Scheduler, only one scheduler is allowed. """ g = ctx.obj Scheduler = load_cls(None, None, scheduler_cls) kwargs = dict(taskdb=g.taskdb, projectdb=g.projectdb, resultdb=g.resultdb, newtask_queue=g.newtask_queue, status_queue=g.status_queue, out_queue=g.scheduler2fetcher, data_path=g.get('data_path', 'data')) if threads: kwargs['threads'] = int(threads) if bloomfilter_on: bloomfilter_config = g.config.get('bloomfilter', {}) bloomfilter_config.setdefault('xmlrpc_host', '127.0.0.1') bloomfilter_config.setdefault('xmlrpc_port', 13100) bloomfilter_rpc = connect_rpc(ctx, None, 'http://%(xmlrpc_host)s:%(xmlrpc_port)s/' % bloomfilter_config) else: bloomfilter_rpc = None scheduler = Scheduler(**kwargs) scheduler.INQUEUE_LIMIT = inqueue_limit scheduler.DELETE_TIME = delete_time scheduler.ACTIVE_TASKS = active_tasks scheduler.LOOP_LIMIT = loop_limit g.instances.append(scheduler) if g.get('testing_mode') or get_object: return scheduler if xmlrpc: utils.run_in_thread(scheduler.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host) scheduler.run()
def setUpClass(self): shutil.rmtree("./data/tests", ignore_errors=True) os.makedirs("./data/tests") run_in_thread(run.run_scheduler, g=run.g) run_in_thread(run.run_fetcher, g=run.g) run_in_thread(run.run_processor, g=run.g) run_in_thread(run.run_result_worker, g=run.g) time.sleep(1) app.config["taskdb"] = run.g.taskdb app.config["projectdb"] = run.g.projectdb app.config["resultdb"] = run.g.resultdb app.config["scheduler_rpc"] = xmlrpclib.ServerProxy("http://localhost:23333") self.app = app.test_client()
def setUpClass(self): shutil.rmtree('./data/tests/', ignore_errors=True) os.makedirs('./data/tests/') def get_resultdb(): return resultdb.ResultDB(self.resultdb_path) self.resultdb = get_resultdb() self.inqueue = Queue(10) def run_result_worker(): self.result_worker = ResultWorker(get_resultdb(), self.inqueue) self.result_worker.run() self.process = run_in_thread(run_result_worker) time.sleep(1)
proxy=proxy, async=async) fetcher.phantomjs_proxy = g.phantomjs_proxy if user_agent: fetcher.user_agent = user_agent if timeout: fetcher.default_options = copy.deepcopy(fetcher.default_options) fetcher.default_options['timeout'] = timeout g.instances.append(fetcher) if g.get('testing_mode'): return fetcher if xmlrpc: utils.run_in_thread(fetcher.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host) fetcher.run() @cli.command() @click.option('--processor-cls', default='pyspider.processor.Processor', callback=load_cls, help='Processor class to be used.') @click.pass_context def processor(ctx, processor_cls, enable_stdout_capture=True): """ Run Processor. """ g = ctx.obj
def setUpClass(self): shutil.rmtree('./data/tests', ignore_errors=True) os.makedirs('./data/tests') import tests.data_test_webpage import httpbin self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887) self.httpbin = 'http://127.0.0.1:14887' ctx = run.cli.make_context('test', [ '--taskdb', 'sqlalchemy+sqlite+taskdb:///data/tests/task.db', '--projectdb', 'sqlalchemy+sqlite+projectdb:///data/tests/projectdb.db', '--resultdb', 'sqlalchemy+sqlite+resultdb:///data/tests/resultdb.db', ], None, obj=ObjectDict(testing_mode=True)) self.ctx = run.cli.invoke(ctx) ctx = run.scheduler.make_context('scheduler', [], self.ctx) scheduler = run.scheduler.invoke(ctx) run_in_thread(scheduler.xmlrpc_run) run_in_thread(scheduler.run) ctx = run.fetcher.make_context('fetcher', [ '--xmlrpc', '--xmlrpc-port', '24444', ], self.ctx) fetcher = run.fetcher.invoke(ctx) run_in_thread(fetcher.xmlrpc_run) run_in_thread(fetcher.run) ctx = run.processor.make_context('processor', [], self.ctx) processor = run.processor.invoke(ctx) run_in_thread(processor.run) ctx = run.result_worker.make_context('result_worker', [], self.ctx) result_worker = run.result_worker.invoke(ctx) run_in_thread(result_worker.run) ctx = run.webui.make_context('webui', [ '--scheduler-rpc', 'http://localhost:23333/' ], self.ctx) app = run.webui.invoke(ctx) app.debug = True self.app = app.test_client() self.rpc = app.config['scheduler_rpc'] time.sleep(1)