def setUpClass(self):
self.projectdb = ProjectDB([
os.path.join(os.path.dirname(__file__),
'data_fetcher_processor_handler.py')
])
self.fetcher = Fetcher(None, None, async=False)
self.status_queue = Queue()
self.newtask_queue = Queue()
self.result_queue = Queue()
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run,
port=14887,
passthrough_errors=False)
self.httpbin = 'http://127.0.0.1:14887'
self.proxy_thread = subprocess.Popen([
'pyproxy', '--username=binux', '--password=123456', '--port=14830',
'--debug'
],
close_fds=True)
self.proxy = '127.0.0.1:14830'
self.processor = Processor(projectdb=self.projectdb,
inqueue=None,
status_queue=self.status_queue,
newtask_queue=self.newtask_queue,
result_queue=self.result_queue)
self.project_name = 'data_fetcher_processor_handler'
time.sleep(0.5)
def run_processor(g=g):
from pyspider.processor import Processor
processor = Processor(projectdb=g.projectdb,
inqueue=g.fetcher2processor,
status_queue=g.status_queue,
newtask_queue=g.newtask_queue,
result_queue=g.processor2result)
g.processor = processor
processor.run()
def processor(ctx, Processor=Processor):
g = ctx.obj
processor = Processor(projectdb=g.projectdb,
inqueue=g.fetcher2processor, status_queue=g.status_queue,
newtask_queue=g.newtask_queue, result_queue=g.processor2result)
g.instances.append(processor)
if g.get('testing_mode'):
return processor
processor.run()
def setUpClass(self):
self.projectdb = ProjectDB([
os.path.join(os.path.dirname(__file__),
'data_fetcher_processor_handler.py')
])
self.fetcher = Fetcher(None, None, async=False)
self.status_queue = Queue()
self.newtask_queue = Queue()
self.result_queue = Queue()
self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run,
port=14887)
self.httpbin = 'http://127.0.0.1:14887'
self.processor = Processor(projectdb=self.projectdb,
inqueue=None,
status_queue=self.status_queue,
newtask_queue=self.newtask_queue,
result_queue=self.result_queue)
self.project_name = 'data_fetcher_processor_handler'
time.sleep(0.5)
