def submit_all(self):
"""
:returns: an IterResult object
"""
if self.num_tasks == 1:
[args] = self.add_task_no(self.task_args, pickle=False)
self.progress('Executing "%s" in process', self.name)
fut = mkfuture(safely_call(self.task_func, args))
return IterResult([fut], self.name, self.num_tasks)
elif self.distribute == 'zmq': # experimental
allargs = self.add_task_no(self.task_args)
w = config.zworkers
it = _starmap(self.task_func, allargs, w.master_host,
w.task_in_port, w.receiver_ports)
ntasks = next(it)
return IterResult(it, self.name, ntasks, self.progress, self.sent)
elif self.distribute == 'qsub': # experimental
allargs = list(self.add_task_no(self.task_args, pickle=False))
logging.warn('Sending %d tasks to the grid engine', len(allargs))
return IterResult(qsub(self.task_func, allargs), self.name,
len(allargs), self.progress, self.sent)
task_no = 0
for args in self.add_task_no(self.task_args):
task_no += 1
self.submit(*args)
if not task_no:
self.progress('No %s tasks were submitted', self.name)
# NB: keep self._iterfutures() an iterator, especially with celery!
ir = IterResult(self._iterfutures(), self.name, task_no, self.progress,
self.sent)
return ir
def main(hostport):
host, port = hostport.split(':')
conn = Client((host, int(port)))
func, args = conn.recv()
res = safely_call(func, args)
try:
conn.send(res)
finally:
conn.close()
def __init__(self, func, iterargs, poolsize=None, progress=logging.info):
self.pool = None
self.func = func
self.name = func.__name__
self.task_args = iterargs
self.progress = progress
self.sent = AccumDict()
self.argnames = inspect.getargspec(func).args
allargs = list(self.add_task_no(iterargs))
progress('Starting %s sequential tasks', self.num_tasks)
self.imap = [safely_call(func, args) for args in allargs]
def submit(self, *args):
"""
Submit a function with the given arguments to the process pool
and add a Future to the list `.results`. If the attribute
distribute is set, the function is run in process and the
result is returned.
"""
check_mem_usage()
# log a warning if too much memory is used
if self.distribute == 'no':
res = safely_call(self.task_func, args)
else:
res = self._submit(args)
self.results.append(res)
