class _async_task(object): """ A class converting blocking functions into asynchronous functions by using threads or processes. """ def __init__(self, func, *args, **kw): self._is_threading = kw.pop('_is_threading') _async_function_counter[func] = _async_function_counter[func] + 1 # specific name name = '<async_task:[%s]<%s|#%s>%s Finished:False>' % \ ('Thread' if self._is_threading else 'Process', _full_func_name(func), _async_function_counter[func], '(%s)' % ';'.join(inspect.signature(func).parameters.keys()), ) self._name = name self._func = func # check initialized pool # ====== using threading ====== # if self._is_threading: global _THREAD_POOL if _THREAD_POOL is None: _THREAD_POOL = ThreadPool(processes=_NTHREADS) self._async_task = _THREAD_POOL.apply_async(func=func, args=args, kwds=kw) # ====== using Multiprocessing ====== # else: def wrapped_func(conn, *args, **kwargs): results = self._func(*args, **kwargs) ret_data = pickle.dumps(results, protocol=pickle.HIGHEST_PROTOCOL) ret_length = len(ret_data) conn.send(ret_length) sent_bytes = 0 while sent_bytes < ret_length: data = ret_data[sent_bytes:sent_bytes + _MAX_PIPE_BLOCK] conn.send(data) sent_bytes += _MAX_PIPE_BLOCK parent_conn, child_conn = Pipe() self._async_task = Process(target=wrapped_func, args=(child_conn, ) + tuple(args), kwargs=kw) self._async_task.start() self._conn = parent_conn @property def name(self): return self._name.replace(':False>', ':%s>' % bool(self.finished)) def __str__(self): return self.name def __repr__(self): return str(self) @property def finished(self): if self._is_threading: return self._async_task.ready() and self._async_task.successful() else: return self._async_task.is_alive() @property def finish(self): return self.get() @property def result(self): return self.get() def get(self, timeout=None): """Return actual result of the function""" if not hasattr(self, '_result'): # ====== multi-threading ====== # if self._is_threading: self._result = self._async_task.get(timeout=timeout) # ====== multi-processing ====== # else: while not self._conn.poll(): # waiting for data pass ret_length = self._conn.recv() read_bytes = 0 received = [] while read_bytes < ret_length: data = self._conn.recv() received.append(data) read_bytes += len(data) received = b"".join(received) self._result = pickle.loads(received) # clean up self._conn.close() self._async_task.join(timeout=timeout) return self._result