def setUpClass(self):
shutil.rmtree('./test/data/', ignore_errors=True)
os.makedirs('./test/data/')
def get_taskdb():
return taskdb.TaskDB(self.taskdb_path)
self.taskdb = get_taskdb()
def get_projectdb():
return projectdb.ProjectDB(self.projectdb_path)
self.projectdb = get_projectdb()
self.newtask_queue = Queue(10)
self.status_queue = Queue(10)
self.scheduler2fetcher = Queue(10)
self.rpc = xmlrpclib.ServerProxy('http://localhost:%d' % self.scheduler_xmlrpc_port)
def run_scheduler():
scheduler = Scheduler(taskdb=get_taskdb(), projectdb=get_projectdb(),
newtask_queue=self.newtask_queue, status_queue=self.status_queue,
out_queue=self.scheduler2fetcher, data_path="./test/data/")
scheduler.UPDATE_PROJECT_INTERVAL = 0.1
scheduler.LOOP_INTERVAL = 0.1
scheduler.INQUEUE_LIMIT = 10
scheduler._last_tick = time.time() # not dispatch cronjob
run_in_thread(scheduler.xmlrpc_run, port=self.scheduler_xmlrpc_port)
scheduler.run()
self.process = run_in_subprocess(run_scheduler)
time.sleep(1)
def all_in_one():
import xmlrpclib
g.scheduler_rpc = xmlrpclib.ServerProxy(
'http://localhost:%d' % g.scheduler_xmlrpc_port)
g.all_in_one = True
threads = []
threads.append(run_in_subprocess(run_result_worker, g=g))
threads.append(run_in_subprocess(run_processor, g=g))
threads.append(run_in_subprocess(run_fetcher, g=g))
threads.append(run_in_subprocess(run_scheduler, g=g))
threads.append(run_in_subprocess(run_webui, g=g))
while True:
try:
time.sleep(10)
except KeyboardInterrupt:
break
for each in threads:
each.join()
def setUpClass(self):
shutil.rmtree('./test/data/', ignore_errors=True)
os.makedirs('./test/data/')
def get_taskdb():
return taskdb.TaskDB(self.taskdb_path)
self.taskdb = get_taskdb()
def get_projectdb():
return projectdb.ProjectDB(self.projectdb_path)
self.projectdb = get_projectdb()
self.newtask_queue = Queue(10)
self.status_queue = Queue(10)
self.scheduler2fetcher = Queue(10)
self.rpc = xmlrpclib.ServerProxy('http://localhost:%d' %
self.scheduler_xmlrpc_port)
def run_scheduler():
scheduler = Scheduler(taskdb=get_taskdb(),
projectdb=get_projectdb(),
newtask_queue=self.newtask_queue,
status_queue=self.status_queue,
out_queue=self.scheduler2fetcher,
data_path="./test/data/")
scheduler.UPDATE_PROJECT_INTERVAL = 0.1
scheduler.LOOP_INTERVAL = 0.1
scheduler._last_tick = time.time() # not dispatch cronjob
run_in_thread(scheduler.xmlrpc_run,
port=self.scheduler_xmlrpc_port)
scheduler.run()
self.process = run_in_subprocess(run_scheduler)
time.sleep(1)
import cPickle as pickle
scheduler_rpc = xmlrpclib.ServerProxy('http://localhost:%d' % scheduler_xmlrpc_port)
fetch_rpc = xmlrpclib.ServerProxy('http://localhost:%d' % fetcher_xmlrpc_port)
from webui.app import app
app.config['fetch'] = lambda task: pickle.loads(fetch_rpc.fetch(task).data)
app.config['taskdb'] = get_taskdb()
app.config['projectdb'] = get_projectdb()
app.config['scheduler_rpc'] = scheduler_rpc
#app.config['cdn'] = '//cdnjs.cloudflare.com/ajax/libs/'
app.run()
if __name__ == '__main__':
if len(sys.argv) < 2:
threads = []
threads.append(run_in_subprocess(run_fetcher))
threads.append(run_in_subprocess(run_processor))
threads.append(run_in_subprocess(run_scheduler))
threads.append(run_in_subprocess(run_webui))
while True:
try:
time.sleep(10)
except KeyboardInterrupt:
break
for each in threads:
each.join()
else:
cmd = "run_"+sys.argv[1]
locals()[cmd]()
fetch_rpc = xmlrpclib.ServerProxy('http://localhost:%d' %
fetcher_xmlrpc_port)
from webui.app import app
app.config['fetch'] = lambda task: pickle.loads(fetch_rpc.fetch(task).data)
app.config['taskdb'] = get_taskdb()
app.config['projectdb'] = get_projectdb()
app.config['scheduler_rpc'] = scheduler_rpc
#app.config['cdn'] = '//cdnjs.cloudflare.com/ajax/libs/'
app.run()
if __name__ == '__main__':
if len(sys.argv) < 2:
threads = []
threads.append(run_in_subprocess(run_fetcher))
threads.append(run_in_subprocess(run_processor))
threads.append(run_in_subprocess(run_scheduler))
threads.append(run_in_subprocess(run_webui))
while True:
try:
time.sleep(10)
except KeyboardInterrupt:
break
for each in threads:
each.join()
else:
cmd = "run_" + sys.argv[1]
locals()[cmd]()
