def get_batch(num_workers, **kwargs):
try:
enqueuer = GeneratorEnqueuer(generator(**kwargs), use_multiprocessing=True)
enqueuer.start(max_queue_size=24, workers=num_workers)
generator_output = None
while True:
while enqueuer.is_running():
if not enqueuer.queue.empty():
generator_output = enqueuer.queue.get()
break
else:
time.sleep(0.01)
yield generator_output
generator_output = None
finally:
if enqueuer is not None:
enqueuer.stop()
def get_batch(num_workers, **kwargs):
try:
enqueuer = GeneratorEnqueuer(generator(**kwargs), use_multiprocessing=True)
print('Generator use 10 batches for buffering, this may take a while, you can tune this yourself.')
enqueuer.start(max_queue_size=10, workers=num_workers)
generator_output = None
while True:
while enqueuer.is_running():
if not enqueuer.queue.empty():
generator_output = enqueuer.queue.get()
break
else:
time.sleep(0.01)
yield generator_output
generator_output = None
finally:
if enqueuer is not None:
enqueuer.stop()
def get_batch(num_workers, **kwargs):
try:
# win10下use_multiprocessing=Fasle，否则报错，linux下use_multiprocessing=True
enqueuer = GeneratorEnqueuer(generator(**kwargs), use_multiprocessing=False)
print('Generator use 10 batches for buffering, this may take a while, you can tune this yourself.')
enqueuer.start(max_queue_size=10, workers=num_workers)
generator_output = None
while True:
while enqueuer.is_running():
if not enqueuer.queue.empty():
generator_output = enqueuer.queue.get()
break
else:
time.sleep(0.01)
yield generator_output
generator_output = None
finally:
if enqueuer is not None:
enqueuer.stop()
def get_batch(num_workers, **kwargs):
enqueuer = None
try:
enqueuer = GeneratorEnqueuer(generator(**kwargs),
use_multiprocessing=True)
enqueuer.start(max_queue_size=64, workers=num_workers)
generator_output = None
while True:
while enqueuer.is_running():
if not enqueuer.queue.empty():
generator_output = enqueuer.queue.get()
break
else:
time.sleep(0.01)
yield generator_output
generator_output = None
finally:
if enqueuer is not None:
enqueuer.stop()
def get_batch(generator, num_workers, use_multiprocessing=True):
try:
enqueuer = GeneratorEnqueuer(generator,
use_multiprocessing=use_multiprocessing)
print(
'Generator use 10 batches for buffering, this may take a while, you can tune this yourself.'
)
enqueuer.start(max_queue_size=20, workers=num_workers)
generator_output = None
while True:
while enqueuer.is_running():
if not enqueuer.queue.empty():
generator_output = enqueuer.queue.get()
break
else:
time.sleep(0.01)
# continue
yield generator_output
generator_output = None
finally:
if enqueuer is not None:
enqueuer.stop()