class Pooler(LocalBase):
def __init__(self, name, subparser, args):
super(Pooler, self).__init__(name)
self.parser = argparse.ArgumentParser(
parents=[subparser],
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
self.parser.add_argument('-c',
'--concurrency',
type=int,
default=20,
help='Size of gevent pool to run jobs with')
self.init_args(args)
## the add methods here and in runner are used not only by the run method
## but also by the scripts themselves when they need to add items back into the input queue
def add(self, item):
self.status_todo += 1
self.pool.spawn(self.run_core, item)
def run(self):
self.input = Queue()
## compared to the multithreaded code, only one thread is running with gevent
## so it is safe to have a callback that calls the respective auxiliary base
self.logger = Queue(callback=self.log_base)
self.errors = Queue(callback=self.err_base)
self.output = Queue(callback=self.out_base)
self.pool = Pool(self.args.concurrency)
self.inn_core()
gevent.monkey.patch_all(thread=False)
if self.phases == [self.run_base]:
self.run_target()
else:
for phase in self.phases:
self.run_base = phase
self.run_target()
self.input.extend(self.to_input)
self.to_input = []
self.status(prefix='\nFinished! -- ')
def run_target(self):
while self.input or not self.seeded.is_set():
my_item = self.input.popleft()
self.pool.spawn(self.run_core, my_item)
self.pool.join()
class Pooler( LocalBase ):
def __init__(self, name, subparser, args):
super(Pooler, self).__init__(name)
self.parser = argparse.ArgumentParser(parents=[subparser],
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
self.parser.add_argument('-c', '--concurrency', type=int, default=20,
help='Size of gevent pool to run jobs with')
self.init_args(args)
## the add methods here and in runner are used not only by the run method
## but also by the scripts themselves when they need to add items back into the input queue
def add(self, item):
self.status_todo += 1
self.pool.spawn(self.run_core, item)
def run(self):
self.input = Queue()
## compared to the multithreaded code, only one thread is running with gevent
## so it is safe to have a callback that calls the respective auxiliary base
self.logger = Queue(callback=self.log_base)
self.errors = Queue(callback=self.err_base)
self.output = Queue(callback=self.out_base)
self.pool = Pool(self.args.concurrency)
self.inn_core()
gevent.monkey.patch_all(thread=False)
if self.phases == [self.run_base]:
self.run_target()
else:
for phase in self.phases:
self.run_base = phase
self.run_target()
self.input.extend(self.to_input)
self.to_input = []
self.status(prefix='\nFinished! -- ')
def run_target(self):
while self.input or not self.seeded.is_set():
my_item = self.input.popleft()
self.pool.spawn(self.run_core, my_item)
self.pool.join()
def run(self):
self.input = Queue()
## compared to the multithreaded code, only one thread is running with gevent
## so it is safe to have a callback that calls the respective auxiliary base
self.logger = Queue(callback=self.log_base)
self.errors = Queue(callback=self.err_base)
self.output = Queue(callback=self.out_base)
self.pool = Pool(self.args.concurrency)
self.inn_core()
gevent.monkey.patch_all(thread=False)
if self.phases == [self.run_base]:
self.run_target()
else:
for phase in self.phases:
self.run_base = phase
self.run_target()
self.input.extend(self.to_input)
self.to_input = []
self.status(prefix='\nFinished! -- ')
def run(self):
self.input = Queue()
## compared to the multithreaded code, only one thread is running with gevent
## so it is safe to have a callback that calls the respective auxiliary base
self.logger = Queue(callback=self.log_base)
self.errors = Queue(callback=self.err_base)
self.output = Queue(callback=self.out_base)
self.pool = Pool(self.args.concurrency)
self.inn_core()
gevent.monkey.patch_all(thread=False)
if self.phases == [self.run_base]:
self.run_target()
else:
for phase in self.phases:
self.run_base = phase
self.run_target()
self.input.extend(self.to_input)
self.to_input = []
self.status(prefix='\nFinished! -- ')