def run_scheduler(g=g):
from pyspider.scheduler import Scheduler
scheduler = Scheduler(taskdb=g.taskdb, projectdb=g.projectdb, resultdb=g.resultdb,
newtask_queue=g.newtask_queue, status_queue=g.status_queue,
out_queue=g.scheduler2fetcher)
g.scheduler = scheduler
run_in_thread(scheduler.xmlrpc_run)
scheduler.run()
def run_scheduler(g=g):
from pyspider.scheduler import Scheduler
scheduler = Scheduler(taskdb=g.taskdb, projectdb=g.projectdb, resultdb=g.resultdb,
newtask_queue=g.newtask_queue, status_queue=g.status_queue,
out_queue=g.scheduler2fetcher)
if g.demo_mode:
scheduler.INQUEUE_LIMIT = 1000
run_in_thread(scheduler.xmlrpc_run, port=g.scheduler_xmlrpc_port, bind=g.webui_host)
scheduler.run()
def run_scheduler(g=g):
from pyspider.scheduler import Scheduler
scheduler = Scheduler(taskdb=g.taskdb,
projectdb=g.projectdb,
resultdb=g.resultdb,
newtask_queue=g.newtask_queue,
status_queue=g.status_queue,
out_queue=g.scheduler2fetcher)
g.scheduler = scheduler
run_in_thread(scheduler.xmlrpc_run)
scheduler.run()
def scheduler(ctx,
xmlrpc,
xmlrpc_host,
xmlrpc_port,
inqueue_limit,
delete_time,
active_tasks,
loop_limit,
Scheduler=Scheduler):
g = ctx.obj
scheduler = Scheduler(taskdb=g.taskdb,
projectdb=g.projectdb,
resultdb=g.resultdb,
newtask_queue=g.newtask_queue,
status_queue=g.status_queue,
out_queue=g.scheduler2fetcher,
data_path=g.get('data_path', 'data'))
scheduler.INQUEUE_LIMIT = inqueue_limit
scheduler.DELETE_TIME = delete_time
scheduler.ACTIVE_TASKS = active_tasks
scheduler.LOOP_LIMIT = loop_limit
g.instances.append(scheduler)
if g.get('testing_mode'):
return scheduler
if xmlrpc:
run_in_thread(scheduler.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host)
scheduler.run()
def run_scheduler(g=g):
from pyspider.scheduler import Scheduler
scheduler = Scheduler(taskdb=g.taskdb,
projectdb=g.projectdb,
resultdb=g.resultdb,
newtask_queue=g.newtask_queue,
status_queue=g.status_queue,
out_queue=g.scheduler2fetcher)
if g.demo_mode:
scheduler.INQUEUE_LIMIT = 1000
run_in_thread(scheduler.xmlrpc_run,
port=g.scheduler_xmlrpc_port,
bind=g.webui_host)
scheduler.run()
def scheduler(ctx, xmlrpc, xmlrpc_host, xmlrpc_port,
inqueue_limit, delete_time, active_tasks, loop_limit, Scheduler=Scheduler):
g = ctx.obj
scheduler = Scheduler(taskdb=g.taskdb, projectdb=g.projectdb, resultdb=g.resultdb,
newtask_queue=g.newtask_queue, status_queue=g.status_queue,
out_queue=g.scheduler2fetcher, data_path=g.get('data_path', 'data'))
scheduler.INQUEUE_LIMIT = inqueue_limit
scheduler.DELETE_TIME = delete_time
scheduler.ACTIVE_TASKS = active_tasks
scheduler.LOOP_LIMIT = loop_limit
g.instances.append(scheduler)
if g.get('testing_mode'):
return scheduler
if xmlrpc:
run_in_thread(scheduler.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host)
scheduler.run()
def scheduler(ctx, xmlrpc, xmlrpc_host, xmlrpc_port,
inqueue_limit, delete_time, active_tasks):
g = ctx.obj
from pyspider.scheduler import Scheduler
scheduler = Scheduler(taskdb=g.taskdb, projectdb=g.projectdb, resultdb=g.resultdb,
newtask_queue=g.newtask_queue, status_queue=g.status_queue,
out_queue=g.scheduler2fetcher)
scheduler.INQUEUE_LIMIT = inqueue_limit
scheduler.DELETE_TIME = delete_time
scheduler.ACTIVE_TASKS = active_tasks
g.instances.append(scheduler)
if xmlrpc:
run_in_thread(scheduler.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host)
scheduler.run()
