def work(data, info):
log = logging.getLogger(info.get('log', 'worker'))
results = []
chunk_size = info.get('chunk_size', 20)
max_workers = info.get('max_workers', 4)
try:
func_str = info.get('worker')
func = load(func_str)
except Exception as exc:
log.error('dynamic worker func invalid! %s' % exc)
return results
Executor = futures.ProcessPoolExecutor
backup_info = deepcopy(info)
with Executor(max_workers=max_workers) as executor:
future_to_data = {}
for index, data_chunked in enumerate(grouper_it(data, chunk_size)):
log.debug('process worker chunk %d processing.' % (index))
info = deepcopy(backup_info)
info['index'] = index
future_to_data[executor.submit(func, data_chunked,
info)] = data_chunked
for future in futures.as_completed(future_to_data):
data = future_to_data[future]
try:
result = future.result()
except Exception as exc:
tt(exc)
log.critical('exception catched! %s %s' % (exc, type(exc)))
result = WorkerException('%s -- %r' % (type(exc), exc))
results.append(result)
return results
async def bound_process(func, data, info):
response = None
log = logging.getLogger(info.get('log', 'worker'))
# log = logging.getLogger('worker')
try:
start_time = time.time()
async with info.get('semaphore'):
response = await func(data, info)
except Exception as exc:
tt(exc)
log.critical('exception catched! %s -- %r' % (exc, data))
response = WorkerException('%s -- %r' % (type(exc), exc))
finally:
end_time = time.time()
log.debug("time elapsed: {}".format(end_time-start_time))
return response
def work(data, info):
log = logging.getLogger(info.get('log', 'worker'))
results = []
chunk_size = info.get('chunk_size', 20)
# max_workers = info.get('max_workers', 4)
try:
func_str = info.get('worker')
func = load(func_str)
except Exception as exc:
log.error('dynamic worker func invalid! %s' % exc)
return results
backup_info = deepcopy(info)
for index, data_chunked in enumerate(grouper_it(data, chunk_size)):
log.debug('simple worker chunk %d processing.' % (index))
info = deepcopy(backup_info)
info['index'] = index
try:
result = func(data_chunked, info)
except Exception as exc:
tt(exc)
log.critical('exception catched! %s %r' % (type(exc), exc))
result = WorkerException('%s -- %r' % (type(exc), exc))
results.append(result)
return results
def worker(self, from_loc, to_loc, start_date, nb_passenger):
raise WorkerException("I'm a fake worker dude !")