def submit_all(self): """ :returns: an IterResult object """ if self.num_tasks == 1: [args] = self.add_task_no(self.task_args, pickle=False) self.progress('Executing "%s" in process', self.name) fut = mkfuture(safely_call(self.task_func, args)) return IterResult([fut], self.name, self.num_tasks) elif self.distribute == 'zmq': # experimental allargs = self.add_task_no(self.task_args) w = config.zworkers it = _starmap(self.task_func, allargs, w.master_host, w.task_in_port, w.receiver_ports) ntasks = next(it) return IterResult(it, self.name, ntasks, self.progress, self.sent) elif self.distribute == 'qsub': # experimental allargs = list(self.add_task_no(self.task_args, pickle=False)) logging.warn('Sending %d tasks to the grid engine', len(allargs)) return IterResult(qsub(self.task_func, allargs), self.name, len(allargs), self.progress, self.sent) task_no = 0 for args in self.add_task_no(self.task_args): task_no += 1 self.submit(*args) if not task_no: self.progress('No %s tasks were submitted', self.name) # NB: keep self._iterfutures() an iterator, especially with celery! ir = IterResult(self._iterfutures(), self.name, task_no, self.progress, self.sent) return ir
def main(hostport): host, port = hostport.split(':') conn = Client((host, int(port))) func, args = conn.recv() res = safely_call(func, args) try: conn.send(res) finally: conn.close()
def __init__(self, func, iterargs, poolsize=None, progress=logging.info): self.pool = None self.func = func self.name = func.__name__ self.task_args = iterargs self.progress = progress self.sent = AccumDict() self.argnames = inspect.getargspec(func).args allargs = list(self.add_task_no(iterargs)) progress('Starting %s sequential tasks', self.num_tasks) self.imap = [safely_call(func, args) for args in allargs]
def submit(self, *args): """ Submit a function with the given arguments to the process pool and add a Future to the list `.results`. If the attribute distribute is set, the function is run in process and the result is returned. """ check_mem_usage() # log a warning if too much memory is used if self.distribute == 'no': res = safely_call(self.task_func, args) else: res = self._submit(args) self.results.append(res)