def work(data, info):
log = logging.getLogger(info.get('log', 'worker'))
results = []
chunk_size = info.get('chunk_size', 20)
max_workers = info.get('max_workers', 4)
try:
func_str = info.get('worker')
func = load(func_str)
except Exception as exc:
log.error('dynamic worker func invalid! %s' % exc)
return results
Executor = futures.ProcessPoolExecutor
backup_info = deepcopy(info)
with Executor(max_workers=max_workers) as executor:
future_to_data = {}
for index, data_chunked in enumerate(grouper_it(data, chunk_size)):
log.debug('process worker chunk %d processing.' % (index))
info = deepcopy(backup_info)
info['index'] = index
future_to_data[executor.submit(func, data_chunked,
info)] = data_chunked
for future in futures.as_completed(future_to_data):
data = future_to_data[future]
try:
result = future.result()
except Exception as exc:
tt(exc)
log.critical('exception catched! %s %s' % (exc, type(exc)))
result = WorkerException('%s -- %r' % (type(exc), exc))
results.append(result)
return results
def work(data, info):
log = logging.getLogger('worker')
begin_time = datetime.now()
response = None
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
tasks = asyncio.ensure_future(_work(data, info))
# signal(SIGINT, lambda s, f: loop.stop())
try:
response = loop.run_until_complete(tasks)
except KeyboardInterrupt as exc:
log.warning('keyboard exited! %s -- %r' % (exc, data))
for task in asyncio.Task.all_tasks():
task.cancel()
except Exception as exc:
loop.stop()
tt(exc)
log.critical(
'exception catched when loop run until compete! %s -- %r' %
(exc, data))
finally:
end_time = datetime.now()
log.debug('coro done. time elapsed: {}'.format(end_time - begin_time))
return response
async def bound_process(func, data, info):
response = None
log = logging.getLogger(info.get('log', 'worker'))
# log = logging.getLogger('worker')
try:
start_time = time.time()
async with info.get('semaphore'):
response = await func(data, info)
except Exception as exc:
tt(exc)
log.critical('exception catched! %s -- %r' % (exc, data))
response = WorkerException('%s -- %r' % (type(exc), exc))
finally:
end_time = time.time()
log.debug("time elapsed: {}".format(end_time-start_time))
return response
async def _work(data, info):
log = logging.getLogger('worker')
chunk_size = info.get('chunk_size', 20)
# max_workers = info.get('max_workers', 4)
max_sem = info.get('max_semaphore', len(data))
try:
func_str = info.get('worker')
func = load(func_str)
except Exception as exc:
log.error('dynamic worker func invalid! %s' % exc)
return None
tasks = []
semaphore = asyncio.Semaphore(max_sem)
async with ClientSession() as session:
backup_info = deepcopy(info)
response = None
for index, data_chunked in enumerate(grouper_it(data, chunk_size)):
info = deepcopy(backup_info)
info['index'] = index
info['session'] = session
info['semaphore'] = semaphore
log.debug('coroutine worker chunk %d processing.' % (index))
task = asyncio.ensure_future(
bound_process(func, data_chunked, info))
tasks.append(task)
try:
gathers = asyncio.gather(*tasks, return_exceptions=False)
except Exception as exc:
tt(exc)
log.critical('exception catched! %s -- %r' % (exc, data))
# response = WorkerException('%s -- %r' % (type(exc), exc))
else:
response = await gathers
finally:
return response
def work(data, info):
log = logging.getLogger(info.get('log', 'worker'))
results = []
chunk_size = info.get('chunk_size', 20)
# max_workers = info.get('max_workers', 4)
try:
func_str = info.get('worker')
func = load(func_str)
except Exception as exc:
log.error('dynamic worker func invalid! %s' % exc)
return results
backup_info = deepcopy(info)
for index, data_chunked in enumerate(grouper_it(data, chunk_size)):
log.debug('simple worker chunk %d processing.' % (index))
info = deepcopy(backup_info)
info['index'] = index
try:
result = func(data_chunked, info)
except Exception as exc:
tt(exc)
log.critical('exception catched! %s %r' % (type(exc), exc))
result = WorkerException('%s -- %r' % (type(exc), exc))
results.append(result)
return results