class _async_task(object):
""" A class converting blocking functions into
asynchronous functions by using threads or processes.
"""
def __init__(self, func, *args, **kw):
self._is_threading = kw.pop('_is_threading')
_async_function_counter[func] = _async_function_counter[func] + 1
# specific name
name = '<async_task:[%s]<%s|#%s>%s Finished:False>' % \
('Thread' if self._is_threading else 'Process',
_full_func_name(func), _async_function_counter[func],
'(%s)' % ';'.join(inspect.signature(func).parameters.keys()),
)
self._name = name
self._func = func
# check initialized pool
# ====== using threading ====== #
if self._is_threading:
global _THREAD_POOL
if _THREAD_POOL is None:
_THREAD_POOL = ThreadPool(processes=_NTHREADS)
self._async_task = _THREAD_POOL.apply_async(func=func,
args=args,
kwds=kw)
# ====== using Multiprocessing ====== #
else:
def wrapped_func(conn, *args, **kwargs):
results = self._func(*args, **kwargs)
ret_data = pickle.dumps(results,
protocol=pickle.HIGHEST_PROTOCOL)
ret_length = len(ret_data)
conn.send(ret_length)
sent_bytes = 0
while sent_bytes < ret_length:
data = ret_data[sent_bytes:sent_bytes + _MAX_PIPE_BLOCK]
conn.send(data)
sent_bytes += _MAX_PIPE_BLOCK
parent_conn, child_conn = Pipe()
self._async_task = Process(target=wrapped_func,
args=(child_conn, ) + tuple(args),
kwargs=kw)
self._async_task.start()
self._conn = parent_conn
@property
def name(self):
return self._name.replace(':False>', ':%s>' % bool(self.finished))
def __str__(self):
return self.name
def __repr__(self):
return str(self)
@property
def finished(self):
if self._is_threading:
return self._async_task.ready() and self._async_task.successful()
else:
return self._async_task.is_alive()
@property
def finish(self):
return self.get()
@property
def result(self):
return self.get()
def get(self, timeout=None):
"""Return actual result of the function"""
if not hasattr(self, '_result'):
# ====== multi-threading ====== #
if self._is_threading:
self._result = self._async_task.get(timeout=timeout)
# ====== multi-processing ====== #
else:
while not self._conn.poll(): # waiting for data
pass
ret_length = self._conn.recv()
read_bytes = 0
received = []
while read_bytes < ret_length:
data = self._conn.recv()
received.append(data)
read_bytes += len(data)
received = b"".join(received)
self._result = pickle.loads(received)
# clean up
self._conn.close()
self._async_task.join(timeout=timeout)
return self._result
